From 9e407e0be01695e7b927f5820ade87ee9602c248 Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Fri, 15 Sep 2017 17:00:14 +0300 Subject: [PATCH] Use epoll API on Linux Also a couple of other modifications due to epoll having a different approach to how the working set of fds is defined and used: 1) open_client() returns an index into the array of clients 2) close_client() has a protection against being called twice with the same client (which would mess up the active_clients counter) Upstream-Status: Submitted [Seebs CC'd by email] Signed-off-by: Alexander Kanavin --- enums/exit_status.in | 3 + pseudo_server.c | 189 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/enums/exit_status.in b/enums/exit_status.in index 6be44d3..88f94cd 100644 --- a/enums/exit_status.in +++ b/enums/exit_status.in @@ -18,3 +18,6 @@ listen_fd, "server loop had no valid listen fd" pseudo_loaded, "server couldn't get out of pseudo environment" pseudo_prefix, "couldn't get valid pseudo prefix" pseudo_invocation, "invalid server command arguments" +epoll_create, "epoll_create() failed" +epoll_ctl, "epoll_ctl() failed" + diff --git a/pseudo_server.c b/pseudo_server.c index ff16efd..14d34de 100644 --- a/pseudo_server.c +++ b/pseudo_server.c @@ -40,6 +40,12 @@ #include "pseudo_client.h" #include "pseudo_db.h" +// This has to come after pseudo includes, as that's where PSEUDO_PORT defines are +#ifdef PSEUDO_PORT_LINUX +#include +#endif + + static int listen_fd = -1; typedef struct { @@ -59,6 +65,7 @@ static int active_clients = 0, highest_client = 0, max_clients = 0; #define LOOP_DELAY 2 #define DEFAULT_PSEUDO_SERVER_TIMEOUT 30 +#define EPOLL_MAX_EVENTS 10 int pseudo_server_timeout = DEFAULT_PSEUDO_SERVER_TIMEOUT; static int die_peacefully = 0; static int die_forcefully = 0; @@ -80,6 +87,9 @@ quit_now(int signal) { static int messages = 0, responses = 0; static struct timeval message_time = { .tv_sec = 0 }; +#ifdef PSEUDO_PORT_LINUX +static void pseudo_server_loop_epoll(void); +#endif static void pseudo_server_loop(void); /* helper function to make a directory, just like mkdir -p. @@ -369,12 +379,16 @@ pseudo_server_start(int daemonize) { kill(ppid, SIGUSR1); } } +#ifdef PSEUDO_PORT_LINUX + pseudo_server_loop_epoll(); +#else pseudo_server_loop(); +#endif return 0; } /* mess with internal tables as needed */ -static void +static unsigned int open_client(int fd) { pseudo_client_t *new_clients; int i; @@ -390,7 +404,7 @@ open_client(int fd) { ++active_clients; if (i > highest_client) highest_client = i; - return; + return i; } } @@ -414,9 +428,11 @@ open_client(int fd) { max_clients += 16; ++active_clients; + return max_clients - 16; } else { pseudo_diag("error allocating new client, fd %d\n", fd); close(fd); + return 0; } } @@ -433,6 +449,10 @@ close_client(int client) { client, highest_client); return; } + if (clients[client].fd == -1) { + pseudo_debug(PDBGF_SERVER, "client %d already closed\n", client); + return; + } close(clients[client].fd); clients[client].fd = -1; free(clients[client].tag); @@ -566,6 +586,171 @@ serve_client(int i) { } } +#ifdef PSEUDO_PORT_LINUX +static void pseudo_server_loop_epoll(void) +{ + struct sockaddr_un client; + socklen_t len; + int i; + int rc; + int fd; + int timeout; + struct epoll_event ev, events[EPOLL_MAX_EVENTS]; + int loop_timeout = pseudo_server_timeout; + + clients = malloc(16 * sizeof(*clients)); + + clients[0].fd = listen_fd; + clients[0].pid = getpid(); + + for (i = 1; i < 16; ++i) { + clients[i].fd = -1; + clients[i].pid = 0; + clients[i].tag = NULL; + clients[i].program = NULL; + } + + active_clients = 1; + max_clients = 16; + highest_client = 0; + + pseudo_debug(PDBGF_SERVER, "server loop started.\n"); + if (listen_fd < 0) { + pseudo_diag("got into loop with no valid listen fd.\n"); + exit(PSEUDO_EXIT_LISTEN_FD); + } + + timeout = LOOP_DELAY * 1000; + + int epollfd = epoll_create1(0); + if (epollfd == -1) { + pseudo_diag("epoll_create1() failed.\n"); + exit(PSEUDO_EXIT_EPOLL_CREATE); + } + ev.events = EPOLLIN; + ev.data.u64 = 0; + if (epoll_ctl(epollfd, EPOLL_CTL_ADD, clients[0].fd, &ev) == -1) { + pseudo_diag("epoll_ctl() failed with listening socket.\n"); + exit(PSEUDO_EXIT_EPOLL_CTL); + } + + pdb_log_msg(SEVERITY_INFO, NULL, NULL, NULL, "server started (pid %d)", getpid()); + + for (;;) { + rc = epoll_wait(epollfd, events, EPOLL_MAX_EVENTS, timeout); + if (rc == 0 || (rc == -1 && errno == EINTR)) { + /* If there's no clients, start timing out. If there + * are active clients, never time out. + */ + if (active_clients == 1) { + loop_timeout -= LOOP_DELAY; + /* maybe flush database to disk */ + pdb_maybe_backup(); + if (loop_timeout <= 0) { + pseudo_debug(PDBGF_SERVER, "no more clients, got bored.\n"); + die_peacefully = 1; + } else { + /* display this if not exiting */ + pseudo_debug(PDBGF_SERVER | PDBGF_BENCHMARK, "%d messages handled in %.4f seconds, %d responses\n", + messages, + (double) message_time.tv_sec + + (double) message_time.tv_usec / 1000000.0, + responses); + } + } + } else if (rc > 0) { + loop_timeout = pseudo_server_timeout; + for (i = 0; i < rc; ++i) { + if (clients[events[i].data.u64].fd == listen_fd) { + if (!die_forcefully) { + len = sizeof(client); + if ((fd = accept(listen_fd, (struct sockaddr *) &client, &len)) != -1) { + /* Don't allow clients to end up on fd 2, because glibc's + * malloc debug uses that fd unconditionally. + */ + if (fd == 2) { + int newfd = fcntl(fd, F_DUPFD, 3); + close(fd); + fd = newfd; + } + pseudo_debug(PDBGF_SERVER, "new client fd %d\n", fd); + /* A new client implicitly cancels any + * previous shutdown request, or a + * shutdown for lack of clients. + */ + pseudo_server_timeout = DEFAULT_PSEUDO_SERVER_TIMEOUT; + die_peacefully = 0; + + ev.events = EPOLLIN; + ev.data.u64 = open_client(fd); + if (ev.data.u64 != 0 && epoll_ctl(epollfd, EPOLL_CTL_ADD, clients[ev.data.u64].fd, &ev) == -1) { + pseudo_diag("epoll_ctl() failed with accepted socket.\n"); + exit(PSEUDO_EXIT_EPOLL_CTL); + } + } else if (errno == EMFILE) { + pseudo_debug(PDBGF_SERVER, "Hit max open files, dropping a client.\n"); + /* In theory there is a potential race here where if we close a client, + it may have sent us a fastop message which we don't act upon. + If we don't close a filehandle we'll loop indefinitely thought. + Only close one per loop iteration in the interests of caution */ + for (int j = 1; j <= highest_client; ++j) { + if (clients[j].fd != -1) { + close_client(j); + break; + } + } + } + } + } else { + struct timeval tv1, tv2; + int rc; + gettimeofday(&tv1, NULL); + rc = serve_client(events[i].data.u64); + gettimeofday(&tv2, NULL); + ++messages; + if (rc == 0) + ++responses; + message_time.tv_sec += (tv2.tv_sec - tv1.tv_sec); + message_time.tv_usec += (tv2.tv_usec - tv1.tv_usec); + if (message_time.tv_usec < 0) { + message_time.tv_usec += 1000000; + --message_time.tv_sec; + } else while (message_time.tv_usec > 1000000) { + message_time.tv_usec -= 1000000; + ++message_time.tv_sec; + } + } + if (die_forcefully) + break; + } + pseudo_debug(PDBGF_SERVER, "server loop complete [%d clients left]\n", active_clients); + } else { + pseudo_diag("epoll_wait failed: %s\n", strerror(errno)); + break; + } + if (die_peacefully || die_forcefully) { + pseudo_debug(PDBGF_SERVER, "quitting.\n"); + pseudo_debug(PDBGF_SERVER | PDBGF_BENCHMARK, "server %d exiting: handled %d messages in %.4f seconds\n", + getpid(), messages, + (double) message_time.tv_sec + + (double) message_time.tv_usec / 1000000.0); + pdb_log_msg(SEVERITY_INFO, NULL, NULL, NULL, "server %d exiting: handled %d messages in %.4f seconds", + getpid(), messages, + (double) message_time.tv_sec + + (double) message_time.tv_usec / 1000000.0); + /* and at this point, we'll start refusing connections */ + close(clients[0].fd); + /* This is a good place to insert a delay for + * debugging race conditions during startup. */ + /* usleep(300000); */ + exit(0); + } + } + +} + +#endif + /* get clients, handle messages, shut down. * This doesn't actually do any work, it just calls a ton of things which * do work. -- 2.14.1