Stream: git-wasmtime

Topic: wasmtime / Issue #2493 Directories with more than ~150 fi...

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 16:32):

abbec opened Issue #2493:

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than 150 files.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 16:32):

abbec labeled Issue #2493:

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than 150 files.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 16:34):

abbec edited Issue #2493:

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than ~150 files.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 16:46):

abbec commented on Issue #2493:

Tested wasmtime versions are 0.19 and 0.21 (both with the same result)

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 19:13):

pchickey assigned Issue #2493:

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than ~150 files.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 19:32):

pchickey commented on Issue #2493:

What toolchain and libc did you compile the example with?

When I compiled with wasi-sdk 12, I was unable to reproduce.
issue2493.zip

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 19:33):

pchickey deleted a comment on Issue #2493:

What toolchain and libc did you compile the example with?

When I compiled with wasi-sdk 12, I was unable to reproduce.
issue2493.zip

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 19:34):

pchickey commented on Issue #2493:

I was able to reproduce, this is a wasi-common bug

view this post on Zulip Wasmtime GitHub notifications bot (Dec 09 2020 at 19:52):

pchickey labeled Issue #2493 (assigned to pchickey):

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than ~150 files.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 10 2020 at 05:48):

sunfishcode commented on Issue #2493:

Me too. #2494 is a fix for this.

view this post on Zulip Wasmtime GitHub notifications bot (Dec 10 2020 at 06:18):

abbec commented on Issue #2493:

Nice, that was fast! :smile:

view this post on Zulip Wasmtime GitHub notifications bot (Dec 11 2020 at 18:52):

sunfishcode closed Issue #2493 (assigned to pchickey):

I am using this simple C program to test:

#include <dirent.h>
#include <stdio.h>
#include <errno.h>

int main(int argc, char **argv) {
  DIR *d;

  char * target = ".";
  if (argc == 2) {
      target = argv[1];
  }
  struct dirent *dir;
  d = opendir(target);
  if (d) {
    while ((dir = readdir(d)) != NULL) {
      printf("%s\n", dir->d_name);
    }

    printf("errno: %d\n", errno);
    closedir(d);
  }

  return(0);
}

I then use this bash script to generate a directory with N files in

rm -rf testfolder
mkdir -p testfolder
for n in $(seq 1 ${1:-100}); do
        touch testfolder/$(printf %04d "$n")
done

This is what happens when I run it

$ bash generate-files.sh 100

$ ls testfolder/
0001  0005  0009  0013  0017  0021  0025  0029  0033  0037  0041  0045  0049  0053  0057  0061  0065  0069  0073  0077  0081  0085  0089  0093  0097
0002  0006  0010  0014  0018  0022  0026  0030  0034  0038  0042  0046  0050  0054  0058  0062  0066  0070  0074  0078  0082  0086  0090  0094  0098
0003  0007  0011  0015  0019  0023  0027  0031  0035  0039  0043  0047  0051  0055  0059  0063  0067  0071  0075  0079  0083  0087  0091  0095  0099
0004  0008  0012  0016  0020  0024  0028  0032  0036  0040  0044  0048  0052  0056  0060  0064  0068  0072  0076  0080  0084  0088  0092  0096  0100

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
103 # 100 files + "." ".." and the "errno" print

$ bash generate-files.sh 200

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | wc -l
147 # lots of files missing

# for example 0012
$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0012

$ wasmtime run ls.wasm --dir=./testfolder -- testfolder | grep 0011
0011

Running wasmer on the same input works as expected:

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | wc -l
201 # wasmer does not return "." and ".."

$ ~/.wasmer/bin/wasmer run ls.wasm --dir=./testfolder -- testfolder | grep 0012
0012

Expected Behavior

Calls to readdir returns all directory entries, even for folders with more than ~150 files.

Last updated: Dec 23 2024 at 12:05 UTC