Binary to hex faster than xxd, part 1 of 2

3 points by textmode 4 years ago | 3 comments

    #include <unistd.h>
    #include <poll.h>
    #include <errno.h>
    #include <sys/types.h>
    #include <sys/stat.h>
    #ifndef EINTR
    #define EINTR (-5004)
    #endif
    #ifndef EAGAIN
    #define EAGAIN (-5011)
    #endif
    #ifndef EWOULDBLOCK
    #define EWOULDBLOCK (-7011)
    #endif
    int writeall(int fd,const void *xv,long long xlen)
    {
      const unsigned char *x = xv;
      long long w;
      while (xlen > 0) {
        w = xlen;
        if (w > 1048576) w = 1048576;
        w = write(fd,x,w);
        if (w < 0) {
          if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
            struct pollfd p;
            p.fd = fd;
            p.events = POLLOUT | POLLERR;
            poll(&p,1,-1);
            continue;
          }
          return -1;
        }
        x += w;
        xlen -= w;
      }
      return 0;
    }
    long long readblock(int fd, void *x, long long xlen) {
        long long r;
        char *buf = x;
        while (xlen > 0) {
            r = xlen;
            if (r > 1048576) r = 1048576;
            r = read(fd, buf, r);
            if (r == 0) break;
            if (r == -1) {
                if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
                    struct pollfd p;
                    p.fd = fd;
                    p.events = POLLIN;
                    poll(&p, 1, -1);
                    continue;
                }
                return -1;
            }
            buf += r; xlen -= r;
        }
        return (buf - (char *)x);
    }
    int fsyncfd(int fd) {
        struct stat st;
        if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
            if (fsync(fd) == -1) return -1;
        }
        return 0;
    }

kazinator 4 years ago |

There is no reason to ever integrate any of these functions into a xxd-like program. Or almost any program.

Non-blocking I/O would ever only be used on pipe-like objects (sockets, serial ports, IPC pipes). There is no reason to use it if you are not multiplexing multiple I/O descriptors onto one thread. There is no point in making a single thread sit in a loop that calls read and write on a single non-blocking descriptor, and uses poll to block. You've just implemented a bunch of complexity in order to do blocking I/O, which you could get by not turning that descriptor non-blocking in the first place.

If you want to bring in POSIX-isms into an xxd-like program, consider mmaping the input (if it is a file) and then treating it as a big binary buffer to hexdump.

I see in your part 2/2 that you're not even calling readblock! The function is unused.

You are using writeall --- for flushing a "char buf [256]" static object when it fills up.

The fsync stuff is pointless; if someone cares, they can call the sync utility in a script after calling your program. Utilities like this should not be second-guessing the I/O subsystem. fsync is for database engines and whatnot. Actually it's pretty useless because it doesn't give hard guarantees. Just because something is flushed out of the operating system's buffers down to the hardware doesn't mean it's actually written into the storage device (disk surface or flash chips or whatever).

Calling fsync can only add a delay before your program terminates, making it look slower, so if you're after writing a fast xxd, why would you be calling it?

textmode 4 years ago | |

writeall() comes from http://nacl.cace-project.eu and there it is used in a networking program. I removed poll() but only see a very small speed increase so far.

fsync() can be removed too but I'm not seeing any resulting speed increase.

Alas, input is not always a file so mmap() is not an option.

textmode 4 years ago |

    #include <unistd.h>
    #include <errno.h>
    #include <sys/types.h>
    int writeall(int fd,const void *xv,long long xlen)
    {
      const unsigned char *x = xv;
      long long w;
      while (xlen > 0) {
        w = xlen;
        if (w > 1048576) w = 1048576;
        w = write(fd,x,w);
        x += w;
        xlen -= w;
      }
      return 0;
    }
    static int hexdigit(char x)
    {
      if (x >= '0' && x <= '9') return x - '0';
      if (x >= 'a' && x <= 'f') return 10 + (x - 'a');
      if (x >= 'A' && x <= 'F') return 10 + (x - 'A');
      return -1;
    }
    int hexparse(unsigned char *y,long long len,const char *x)
    {
      if (!x) return 0;
      while (len > 0) {
        int digit0;
        int digit1;
        digit0 = hexdigit(x[0]); if (digit0 == -1) return 0;
        digit1 = hexdigit(x[1]); if (digit1 == -1) return 0;
        *y++ = digit1 + 16 * digit0;
        --len;
        x += 2;
      }
      if (x[0]) return 0;
      return 1;
    }