From 9e171b0b16591d16bc7d9efc2fc8e376371b97ba Mon Sep 17 00:00:00 2001 From: Reetesh Ranjan Date: Sat, 11 Jul 2015 00:57:07 +0530 Subject: [PATCH 1/3] File Descriptor Leak through Import The 'fd' in do_import() was leaked. It caused an issue in a deployment where we were updating the data source every 15 minutes and tried to use the 'import' API to update the data source (without dumping the existing data). The fix is to have a global 'if_fd' in place of the local 'fd' and close it before opening the file again in do_import. --- src/main.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 3498517..b7bf80a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -52,6 +52,7 @@ PhraseMap pm; // Phrase Map (usually a sorted array of strings RMQ st; // An instance of the RMQ Data Structure char *if_mmap_addr = NULL; // Pointer to the mmapped area of the file off_t if_length = 0; // The length of the input file +int if_fd = -1; // Fix for file descriptor leaks volatile bool building = false; // TRUE if the structure is being built unsigned long nreq = 0; // The total number of requests served till now int line_limit = -1; // The number of lines to import from the input file @@ -485,11 +486,16 @@ do_import(std::string file, uint_t limit, FILE *fin = fopen(file.c_str(), "r"); #endif - int fd = open(file.c_str(), O_RDONLY); + if(-1 != if_fd) { + close(if_fd); + if_fd = -1; + } + + if_fd = open(file.c_str(), O_RDONLY); - DCERR("handle_import::file:" << file << "[fin: " << (!!fin) << ", fd: " << fd << "]" << endl); + DCERR("handle_import::file:" << file << "[fin: " << (!!fin) << ", fd: " << if_fd << "]" << endl); - if (!fin || fd == -1) { + if (!fin || if_fd == -1) { perror("fopen"); return -IMPORT_FILE_NOT_FOUND; } @@ -511,12 +517,12 @@ do_import(std::string file, uint_t limit, if_length = file_size(file.c_str()); // mmap() the input file in - if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, fd, 0); + if_mmap_addr = (char*)mmap(NULL, if_length, PROT_READ, MAP_SHARED, if_fd, 0); if (if_mmap_addr == MAP_FAILED) { - fprintf(stderr, "length: %llu, fd: %d\n", if_length, fd); + fprintf(stderr, "length: %llu, fd: %d\n", if_length, if_fd); perror("mmap"); if (fin) { fclose(fin); } - if (fd != -1) { close(fd); } + if (if_fd != -1) { close(if_fd); } building = false; return -IMPORT_MMAP_FAILED; } From 017dec7f81149b1936c52ea107d5d008fb0ef2f0 Mon Sep 17 00:00:00 2001 From: Reetesh Ranjan Date: Thu, 16 Jul 2015 00:41:26 +0530 Subject: [PATCH 2/3] Auto restart using forks and watching in parent --- src/main.cpp | 50 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index b7bf80a..cb12560 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -817,18 +818,9 @@ parse_options(int argc, char *argv[]) { break; } } - - } - -int -main(int argc, char* argv[]) { - parse_options(argc, argv); - if (opt_show_help) { - show_usage(argv); - return 0; - } +int do_server(void) { started_at = time(NULL); @@ -870,3 +862,41 @@ main(int argc, char* argv[]) { } return 0; } + +int +main(int argc, char* argv[]) { + parse_options(argc, argv); + if (opt_show_help) { + show_usage(argv); + return 0; + } + + int result = fork(); + + if(result == 0) { + do_server(); + } + else if(result < 0) { + perror("fork: "); exit(1); + } + else { + for(;;) + { + int status = 0; + waitpid(-1, &status, 0); + if(!WIFEXITED(status)) + { + result = fork(); + if(result == 0) { + do_server(); + } + else if(result < 0) + { + puts("Crashed and cannot restart"); + exit(1); + } + } + else exit(0); + } + } +} From 8eb5f68aedf5f6d6aac64903bdbe92a8554d9275 Mon Sep 17 00:00:00 2001 From: Reetesh Ranjan Date: Mon, 14 Mar 2016 20:29:40 +0530 Subject: [PATCH 3/3] Max line size needs to be bigger --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index cb12560..35e3aa7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -37,7 +37,7 @@ #if !defined INPUT_LINE_SIZE // Max. line size is 8191 bytes. -#define INPUT_LINE_SIZE 8192 +#define INPUT_LINE_SIZE 32768 #endif // How many bytes to reserve for the output string