The anatomy of tee program on OpenBSD

The tee command is used to read content from standard input and displays it not only in standard output but also saves to other files simultaneously. The source code of tee in OpenBSD is very simple, and I want to give it an analysis:

(1) tee leverages Singlely-linked List defined in sys/queue.h to manage outputted files (including standard output):

struct list {
    SLIST_ENTRY(list) next;
    int fd;
    char *name;
};
SLIST_HEAD(, list) head;

......

static void
add(int fd, char *name)
{
    struct list *p;
    ......
    SLIST_INSERT_HEAD(&head, p, next);
}

int
main(int argc, char *argv[])
{
    struct list *p;
    ......
    SLIST_INIT(&head);
    ......
    SLIST_FOREACH(p, &head, next) {
        ......
    }
}

To understand it easily, I extract the macros from sys/queue.h and created a file which utilizes the marcos:

#define SLIST_HEAD(name, type)                      \
struct name {                               \
    struct type *slh_first; /* first element */         \
}

#define SLIST_ENTRY(type)                       \
struct {                                \
    struct type *sle_next;  /* next element */          \
}

#define SLIST_FIRST(head)   ((head)->slh_first)
#define SLIST_END(head)     NULL
#define SLIST_EMPTY(head)   (SLIST_FIRST(head) == SLIST_END(head))
#define SLIST_NEXT(elm, field)  ((elm)->field.sle_next)

#define SLIST_FOREACH(var, head, field)                 \
    for((var) = SLIST_FIRST(head);                  \
        (var) != SLIST_END(head);                   \
        (var) = SLIST_NEXT(var, field))

#define SLIST_INIT(head) {                      \
    SLIST_FIRST(head) = SLIST_END(head);                \
}

#define SLIST_INSERT_HEAD(head, elm, field) do {            \
    (elm)->field.sle_next = (head)->slh_first;          \
    (head)->slh_first = (elm);                  \
} while (0)

struct list {
    SLIST_ENTRY(list) next;
    int fd;
    char *name;
};
SLIST_HEAD(, list) head;

int
main(int argc, char *argv[])
{
    struct list *p;
    SLIST_INIT(&head);

    SLIST_INSERT_HEAD(&head, p, next);
    SLIST_FOREACH(p, &head, next) {

    }
}

Then employed gcc‘s pre-processing function:

# gcc -E slist.c
# 1 "slist.c"
# 1 "<built-in>"
# 1 "<command-line>"
# 1 "slist.c"
# 30 "slist.c"
struct list {
 struct { struct list *sle_next; } next;
 int fd;
 char *name;
};
struct { struct list *slh_first; } head;

int
main(int argc, char *argv[])
{
 struct list *p;
 { ((&head)->slh_first) = NULL; };

 do { (p)->next.sle_next = (&head)->slh_first; (&head)->slh_first = (p); } while (0);
 for((p) = ((&head)->slh_first); (p) != NULL; (p) = ((p)->next.sle_next)) {

 }
}

It becomes clear now! The head node in list contains only 1 member: slh_first, which points to the first valid node. For the elements in the list, it is embedded with next struct which uses sle_next to refer to next buddy.

(2) By default, tee will overwrite the output files. If you want to append it, use -a option, and the code is as following:

while (*argv) {
    if ((fd = open(*argv, O_WRONLY | O_CREAT |
        (append ? O_APPEND : O_TRUNC), DEFFILEMODE)) == -1) {
        ......
    } 
    ......
}

(3) The next part is the skeleton of saving content to files:

while ((rval = read(STDIN_FILENO, buf, sizeof(buf))) > 0) {
    SLIST_FOREACH(p, &head, next) {
        n = rval;
        bp = buf;
        do {
            if ((wval = write(p->fd, bp, n)) == -1) {
                ......
            }
            bp += wval;
        } while (n -= wval);
    }
}

We need to iterates every opened file descriptor and write contents into it.

(4) Normally, theinterrupt signal will cause tee exit:

# tee
fdkfkdfjk
fdkfkdfjk
^C
#

To disable this feature, use -i option:

# tee -i
fdhfhd
fdhfhd
^C^C

The corresponding code is like this:

......
case 'i':
    (void)signal(SIGINT, SIG_IGN);
    break;

The gRPC server program will crash when can’t bind successfully

a server program which uses gRPC crashed when started:

$ ./server 60001
......
E1123 15:37:54.133480971   14408 server_chttp2.c:38]         {"created":"@1511422674.133408109","description":"No address added out of total 1 resolved","file":"src/core/ext/transport/chttp2/server/chttp2_server.c","file_line":245,"referenced_errors":[{"created":"@1511422674.133405147","description":"Failed to add any wildcard listeners","file":"src/core/lib/iomgr/tcp_server_posix.c","file_line":338,"referenced_errors":[{"created":"@1511422674.133394827","description":"Unable to configure socket","fd":4,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c","file_line":200,"referenced_errors":[{"created":"@1511422674.133385167","description":"OS Error","errno":98,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c","file_line":173,"os_error":"Address already in use","syscall":"bind"}]},{"created":"@1511422674.133404647","description":"Unable to configure socket","fd":4,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c","file_line":200,"referenced_errors":[{"created":"@1511422674.133401558","description":"OS Error","errno":98,"file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c","file_line":173,"os_error":"Address already in use","syscall":"bind"}]}]}]}
Segmentation fault (core dumped)

It is weird because it runs well yesterday. Check the core dump file:

......
Core was generated by `./server 60001'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00007f30edbbf9b0 in pthread_mutex_lock () from /usr/lib/libpthread.so.0
(gdb) bt
#0  0x00007f30edbbf9b0 in pthread_mutex_lock () from /usr/lib/libpthread.so.0
#1  0x000055c312c87077 in grpc::Server::Wait() ()
......

No clue. So format the above json log:

{  
   "created":"@1511422674.133408109",
   "description":"No address added out of total 1 resolved",
   "file":"src/core/ext/transport/chttp2/server/chttp2_server.c",
   "file_line":245,
   "referenced_errors":[  
      {  
         "created":"@1511422674.133405147",
         "description":"Failed to add any wildcard listeners",
         "file":"src/core/lib/iomgr/tcp_server_posix.c",
         "file_line":338,
         "referenced_errors":[  
            {  
               "created":"@1511422674.133394827",
               "description":"Unable to configure socket",
               "fd":4,
               "file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c",
               "file_line":200,
               "referenced_errors":[  
                  {  
                     "created":"@1511422674.133385167",
                     "description":"OS Error",
                     "errno":98,
                     "file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c",
                     "file_line":173,
                     "os_error":"Address already in use",
                     "syscall":"bind"
                  }
               ]
            },
            {  
               "created":"@1511422674.133404647",
               "description":"Unable to configure socket",
               "fd":4,
               "file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c",
               "file_line":200,
               "referenced_errors":[  
                  {  
                     "created":"@1511422674.133401558",
                     "description":"OS Error",
                     "errno":98,
                     "file":"src/core/lib/iomgr/tcp_server_utils_posix_common.c",
                     "file_line":173,
                     "os_error":"Address already in use",
                     "syscall":"bind"
                  }
               ]
            }
         ]
      }

Oh, I see. “Address already in use” means the 60001 port is occupied already. I switch to another port, and it works.

Build SEAL on Linux

Building SEAL(Simple Encrypted Arithmetic Library) on Linux needs some tweaks:

(1) Add executable attribute for configure file:

$ chmod a+x configure

(2) Running configure generates following errors:

$ ./configure
-bash: ./configure: /bin/sh^M: bad interpreter: No such file or directory

dos2unix doesn’t take effect too:

$ dos2unix configure
dos2unix: Binary symbol 0x07 found at line 3911
dos2unix: Skipping binary file configure

Need tr to save me:

$ tr -d '\r' < configure > temp
$ mv temp configure

Then compiling is OK:

$ ./configure
$ make

The difference of loopback packets on Linux and OpenBSD

Capture the packets on loopback network card on Linux:

# tcpdump -i lo -w lo.pcap port 33333
tcpdump: listening on lo, link-type EN10MB (Ethernet), capture size 262144 bytes
......

Download it onto Windows and use wireshark to analyze it:

1

We can see every packet conforms to standard ethernet format.

Capture lookback packets on OpenBSD:

# tcpdump -i lo0 -w lo.pcap port 33333
tcpdump: listening on lo0, link-type LOOP
......

Also download it onto Windows and open it with wireshark:

2

The wireshark just recognizes the packet as “Raw IP” format, but can’t show details.

After referring discussion in Wireshark mailing list, I know it is related to network link-layer header type0x0C stands for “Raw IP”:

3

I modified the 0x0C to 0x6C, which means “OpenBSD loopback”:

4

Now the packets can be decoded successfully:

5

P.S., I also started a discussion about this issue in mailing list.

Update: I write a script to do this conversion.

The gRPC will terminate process when it can’t allocate memory

My program leverages gRPC, and after a stress testing, it crashed. Use gdb to debug the core dump file:

[Current thread is 1 (Thread 0x7f73ef5f1780 (LWP 147393))]
(gdb) bt
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1  0x00007f73edee542a in __GI_abort () at abort.c:89
#2  0x00005580f74da93a in gpr_malloc ()
#3  0x00005580f74e4b65 in ?? ()
#4  0x00005580f74e28d4 in grpc_exec_ctx_flush ()
#5  0x00005580f758aee1 in ?? ()
#6  0x00005580f75061b7 in grpc_pollset_work ()
#7  0x00005580f74f1c78 in ?? ()
#8  0x00005580f72ab85e in grpc::CompletionQueue::Pluck (this=0x5580f96b3698, tag=0x7fff9cafbe90)
......

I doubt the root cause should be memory is not enough, but not sure. Check the source code of gpr_malloc:

void* gpr_malloc(size_t size) {
  void* p;
  if (size == 0) return nullptr;
  GPR_TIMER_BEGIN("gpr_malloc", 0);
  p = g_alloc_functions.malloc_fn(size);
  if (!p) {
    abort();
  }
  GPR_TIMER_END("gpr_malloc", 0);
  return p;
}

We can see if p is NULL, the abort() system call will be invoked. This verifies what I guessed. Besides gpr_malloc, other memory allocate functions (such as gpr_zalloc, gpr_realloc, etc) have the same behaviors.