diff --git a/socket.c b/socket.c
index 843c19b6..c4420ba7 100644
--- a/socket.c
+++ b/socket.c
@@ -435,9 +435,9 @@ void start_accept_loop(int port, int (*fn)(int ))
 			 * overloaded.  Sleep briefly before trying to
 			 * accept again. */
 			sleep(2);
+		} else {
+			close(fd);
 		}
-
-		close(fd);
 	}
 }