From db8a382eb33a4eed04698bda06660b2b91d7b4da Mon Sep 17 00:00:00 2001 From: "aleksandr.korepanov" Date: Fri, 25 Jan 2019 13:52:06 +0300 Subject: [PATCH] Add PCH supporting. --- include_server/test_data/distcc/src/compile.h | 2 + include_server/test_data/distcc/src/distcc.h | 5 +- include_server/test_data/distcc/src/util.h | 1 + src/compile.c | 8 +- src/compile.h | 2 + src/cpp.c | 27 +++- src/distcc.h | 5 +- src/lock.c | 4 +- src/lock.h | 2 + src/remote.c | 115 +++++++++++++++++- src/serve.c | 79 ++++++++++++ src/tempfile.c | 35 ++++++ src/util.c | 12 +- src/util.h | 1 + 14 files changed, 285 insertions(+), 13 deletions(-) diff --git a/include_server/test_data/distcc/src/compile.h b/include_server/test_data/distcc/src/compile.h index 6876bab7..d8b42ba2 100644 --- a/include_server/test_data/distcc/src/compile.h +++ b/include_server/test_data/distcc/src/compile.h @@ -24,11 +24,13 @@ int dcc_compile_remote(char **argv, char *input_fname, char *cpp_fname, + char *header_list_fname, char **file_names, char *output_fname, char *deps_fname, char *server_stderr_fname, pid_t cpp_pid, + pid_t header_list_pid, int local_cpu_lock_fd, struct dcc_hostdef *host, int *status); diff --git a/include_server/test_data/distcc/src/distcc.h b/include_server/test_data/distcc/src/distcc.h index c0686a70..48139d99 100644 --- a/include_server/test_data/distcc/src/distcc.h +++ b/include_server/test_data/distcc/src/distcc.h @@ -250,6 +250,7 @@ int dcc_get_tempdir(const char **); int dcc_make_tmpnam(const char *, const char *suffix, char **); int dcc_get_new_tmpdir(char **tmpdir); int dcc_mk_tmpdir(const char *path); +int dcc_mkdir_recursively(const char *path); int dcc_mkdir(const char *path); int dcc_get_lock_dir(char **path_ret) WARN_UNUSED; @@ -269,8 +270,8 @@ int dcc_strip_local_args(char **from, char ***out_argv); int dcc_strip_dasho(char **from, char ***out_argv); /* cpp.c */ -int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, - pid_t *cpp_pid); +int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, pid_t *cpp_pid); +int dcc_depends_header_list(char **argv, char *input_fname, char **header_list_fname, pid_t *header_list_pid); /* filename.c */ int dcc_is_source(const char *sfile); diff --git a/include_server/test_data/distcc/src/util.h b/include_server/test_data/distcc/src/util.h index e2aefed6..bb2a473f 100644 --- a/include_server/test_data/distcc/src/util.h +++ b/include_server/test_data/distcc/src/util.h @@ -34,6 +34,7 @@ int dcc_getenv_bool(const char *name, int def_value); int set_cloexec_flag (int desc, int value); int dcc_ignore_sigpipe(int val); int dcc_remove_if_exists(const char *fname); +int dcc_rename(const char *old_filename, const char *new_filename); int dcc_trim_path(const char *compiler_name); int dcc_set_path(const char *newpath); char *dcc_abspath(const char *path, int path_len); diff --git a/src/compile.c b/src/compile.c index e45fba52..939c7385 100644 --- a/src/compile.c +++ b/src/compile.c @@ -653,6 +653,7 @@ dcc_build_somewhere(char *argv[], int *status) { char *input_fname = NULL, *output_fname, *cpp_fname, *deps_fname = NULL; + char *header_list_fname = NULL; char **files; char **server_side_argv = NULL; int server_side_argv_deep_copied = 0; @@ -660,6 +661,7 @@ dcc_build_somewhere(char *argv[], int needs_dotd = 0; int sets_dotd_target = 0; pid_t cpp_pid = 0; + pid_t header_list_pid = 0; int cpu_lock_fd = -1, local_cpu_lock_fd = -1; int ret; int remote_ret = 0; @@ -781,6 +783,9 @@ dcc_build_somewhere(char *argv[], if ((ret = dcc_cpp_maybe(argv, input_fname, &cpp_fname, &cpp_pid) != 0)) goto fallback; + if ((ret = dcc_depends_header_list(argv, input_fname, &header_list_fname, &header_list_pid))) + goto fallback; + if ((ret = dcc_strip_local_args(argv, &server_side_argv))) goto fallback; @@ -804,11 +809,12 @@ dcc_build_somewhere(char *argv[], if ((ret = dcc_compile_remote(server_side_argv, input_fname, cpp_fname, + header_list_fname, files, output_fname, needs_dotd ? deps_fname : NULL, server_stderr_fname, - cpp_pid, local_cpu_lock_fd, + cpp_pid, header_list_pid, local_cpu_lock_fd, host, status)) != 0) { /* Returns zero if we successfully ran the compiler, even if * the compiler itself bombed out. */ diff --git a/src/compile.h b/src/compile.h index 5eaaf49d..aff393d8 100644 --- a/src/compile.h +++ b/src/compile.h @@ -26,11 +26,13 @@ int dcc_compile_remote(char **argv, char *input_fname, char *cpp_fname, + char *header_list_fname, char **file_names, char *output_fname, char *deps_fname, char *server_stderr_fname, pid_t cpp_pid, + pid_t header_list_pid, int local_cpu_lock_fd, struct dcc_hostdef *host, int *status); diff --git a/src/cpp.c b/src/cpp.c index 8f059860..986ddb49 100644 --- a/src/cpp.c +++ b/src/cpp.c @@ -51,8 +51,7 @@ * allows us to overlap opening the TCP socket, which probably doesn't * use many cycles, with running the preprocessor. **/ -int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, - pid_t *cpp_pid) +int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, pid_t *cpp_pid) { char **cpp_argv; int ret; @@ -99,3 +98,27 @@ int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, return dcc_spawn_child(cpp_argv, cpp_pid, "/dev/null", *cpp_fname, NULL); } + +int dcc_depends_header_list(char **argv, char *input_fname, char **header_list_fname, pid_t *header_list_pid) +{ + char **header_list_argv; + int ret; + + *header_list_pid = 0; + + if (dcc_is_preprocessed(input_fname)) { + rs_log_error("failed to get depends header list, file already preprocessed"); + return EXIT_DISTCC_FAILED; + } + + if ((ret = dcc_make_tmpnam("distcc", ".list", header_list_fname))) + return ret; + + if ((ret = dcc_strip_dasho(argv, &header_list_argv)) + || (ret = dcc_set_action_opt(header_list_argv, "-MM"))) + return ret; + + /* FIXME: header_list_argv is leaked */ + return dcc_spawn_child(header_list_argv, header_list_pid, + "/dev/null", *header_list_fname, NULL); +} diff --git a/src/distcc.h b/src/distcc.h index 23f9456e..12eb50ac 100644 --- a/src/distcc.h +++ b/src/distcc.h @@ -272,6 +272,7 @@ int dcc_get_tempdir(const char **); int dcc_make_tmpnam(const char *, const char *suffix, char **); int dcc_get_new_tmpdir(char **tmpdir); int dcc_mk_tmpdir(const char *path); +int dcc_mkdir_recursively(const char *path); int dcc_mkdir(const char *path); int dcc_get_subdir(const char *name, char **path_ret) WARN_UNUSED; @@ -292,8 +293,8 @@ int dcc_strip_local_args(char **from, char ***out_argv); int dcc_strip_dasho(char **from, char ***out_argv); /* cpp.c */ -int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, - pid_t *cpp_pid); +int dcc_cpp_maybe(char **argv, char *input_fname, char **cpp_fname, pid_t *cpp_pid); +int dcc_depends_header_list(char **argv, char *input_fname, char **header_list_fname, pid_t *header_list_pid); /* filename.c */ int dcc_is_source(const char *sfile); diff --git a/src/lock.c b/src/lock.c index 74d086d6..3dbb51eb 100644 --- a/src/lock.c +++ b/src/lock.c @@ -176,7 +176,7 @@ int dcc_make_lock_filename(const char *lockname, * @retval 0 if we got the lock * @retval -1 with errno set if the file is already locked. **/ -static int sys_lock(int fd, int block) +int dcc_lock(int fd, int block) { #if defined(F_SETLK) struct flock lockparam; @@ -291,7 +291,7 @@ int dcc_lock_host(const char *lockname, return ret; } - if (sys_lock(*lock_fd, block) == 0) { + if (dcc_lock(*lock_fd, block) == 0) { rs_trace("got %s lock on %s slot %d as fd%d", lockname, host->hostdef_string, slot, *lock_fd); free(fname); diff --git a/src/lock.h b/src/lock.h index 35ac7805..625c23a3 100644 --- a/src/lock.h +++ b/src/lock.h @@ -24,6 +24,8 @@ int dcc_lock_host(const char *lockname, const struct dcc_hostdef *host, int slot, int block, int *lock_fd); +int dcc_lock(int fd, int block); + int dcc_unlock(int lock_fd); int dcc_make_lock_filename(const char *lockname, diff --git a/src/remote.c b/src/remote.c index 2d00e82a..fbbfc4a4 100644 --- a/src/remote.c +++ b/src/remote.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -149,7 +150,98 @@ dcc_send_header(int net_fd, return 0; } +static int +dcc_get_precompiled_header_path(const char *header_list_fname, char precompiled_header_path[PATH_MAX]) { + int ret; + int fd; + off_t file_size; + + precompiled_header_path[0] = 0; + if (header_list_fname == NULL) { + return 0; + } + + if ((ret = dcc_open_read(header_list_fname, &fd, &file_size))) { + return ret; + } + + char *data = (char *)malloc(file_size + 1); + if (data == NULL) { + dcc_close(fd); + return EXIT_OUT_OF_MEMORY; + } + + if ((ret = dcc_readx(fd, data, file_size))) { + free(data); + dcc_close(fd); + return ret; + } + data[file_size] = 0; + + const char precompiled_preprocess[] = "#pragma GCC pch_preprocess \""; + const char *begin = strstr(data, precompiled_preprocess); + if (begin) { + begin += strlen(precompiled_preprocess); + const char *end = strchr(begin, '"'); + if (!end || end - begin >= PATH_MAX) { + free(data); + dcc_close(fd); + return EXIT_DISTCC_FAILED; + } + strncpy(precompiled_header_path, begin, end - begin); + precompiled_header_path[end - begin] = 0; + } + + free(data); + dcc_close(fd); + return 0; +} + +/** + * Sends precompiled header path and sends precompiled header itself if need. + * + * PCHP: precompiled header path on disk; it should be present on distccd agent + * PCHR: + * 0 - if precompiled header is present on distccd agent or we do not need it at all + * 1 - if precompiled header is absent on distccd agent and we should send it + * PCHF: precompiled header file itself + **/ +static int +try_exchange_precompiled_header(int to_net_fd, int from_net_fd, char *input_fname, int *status, + const char *header_list_fname, pid_t header_list_pid) { + int ret; + if ((ret = dcc_wait_for_cpp(header_list_pid, status, input_fname))) { + return ret; + } + + char precompiled_header_path[PATH_MAX] = {0}; + if ((ret = dcc_get_precompiled_header_path(header_list_fname, precompiled_header_path))) { + return ret; + } + /* expect that precompiled header is placed on tmp dir */ + /* TODO: may be such limitation too strict? */ + if (precompiled_header_path[0] && strncmp(precompiled_header_path, "/tmp/", 5)) { + rs_log_warning("expect precompiled header in /tmp dir, got %s", precompiled_header_path); + precompiled_header_path[0] = 0; + } + + if ((ret = dcc_x_token_string(to_net_fd, "PCHP", precompiled_header_path))) { + return ret; + } + + unsigned precompiled_header_required; + if ((ret = dcc_r_token_int(from_net_fd, "PCHR", &precompiled_header_required))) { + return ret; + } + + if (precompiled_header_required) { + if ((ret = dcc_x_file(to_net_fd, precompiled_header_path, "PCHF", DCC_COMPRESS_LZO1X, NULL))) { + return ret; + } + } + return 0; +} /** * Pass a compilation across the network. * @@ -164,17 +256,23 @@ dcc_send_header(int net_fd, * * @param argv Compiler command to run. * - * @param cpp_fname Filename of preprocessed source. May not be complete yet, + * @param cpp_fname Filename of preprocessed source. May not be complete yet, * depending on @p cpp_pid. * + * @param header_list_fname Filename of list with used headers. + * May not be complete yet, depending on @p header_list_pid. + * * @param files If we are doing preprocessing on the server, the names of * all the files needed; otherwise, NULL. * * @param output_fname File that the object code should be delivered to. * - * @param cpp_pid If nonzero, the pid of the preprocessor. Must be + * @param cpp_pid If nonzero, the pid of the preprocessor. Must be * allowed to complete before we send the input file. * + * @param header_list_pid If nonzero, the pid of the preprocessor for getting used headers. + * Must be allowed to complete before we send the input file. + * * @param local_cpu_lock_fd If != -1, file descriptor for the lock file. * Should be != -1 iff (host->cpp_where != DCC_CPP_ON_SERVER). * If != -1, the lock must be held on entry to this function, @@ -195,11 +293,13 @@ dcc_send_header(int net_fd, int dcc_compile_remote(char **argv, char *input_fname, char *cpp_fname, + char *header_list_fname, char **files, char *output_fname, char *deps_fname, char *server_stderr_fname, pid_t cpp_pid, + pid_t header_list_pid, int local_cpu_lock_fd, struct dcc_hostdef *host, int *status) @@ -251,6 +351,12 @@ int dcc_compile_remote(char **argv, goto out; } + /* Here will sent just an empty string */ + if ((ret = try_exchange_precompiled_header(to_net_fd, from_net_fd, input_fname, status, + header_list_fname, header_list_pid))) { + goto out; + } + n_files = dcc_argv_len(files); if ((ret = dcc_x_many_files(to_net_fd, n_files, files))) { goto out; @@ -263,6 +369,11 @@ int dcc_compile_remote(char **argv, if ((ret = dcc_send_header(to_net_fd, argv, host))) goto out; + if ((ret = try_exchange_precompiled_header(to_net_fd, from_net_fd, input_fname, status, + header_list_fname, header_list_pid))) { + goto out; + } + if ((ret = dcc_wait_for_cpp(cpp_pid, status, input_fname))) goto out; diff --git a/src/serve.c b/src/serve.c index 81e50a1d..6755a61c 100644 --- a/src/serve.c +++ b/src/serve.c @@ -86,6 +86,7 @@ #include "stringmap.h" #include "dotd.h" #include "fix_debug_info.h" +#include "lock.h" #ifdef HAVE_GSSAPI #include "auth.h" @@ -630,6 +631,80 @@ static int make_temp_dir_and_chdir_for_cpp(int in_fd, return ret; } +/** + * Accepts precompiled header path, checks if it exist on disk and loads if need. + * + * PCHP: precompiled header path on disk; it should be present on distccd agent + * PCHR: + * 0 - if precompiled header is present on distccd agent or we do not need it at all + * 1 - if precompiled header is absent on distccd agent and we should sent it + * PCHF: precompiled header file itself + **/ +static int try_load_precompiled_header (int in_fd, int out_fd) { + int ret = 0; + int lock_fd = 0; + char *precompiled_header_path = NULL; + char *precompiled_header_lock = NULL; + + do { + if ((ret = dcc_r_token_string(in_fd, "PCHP", &precompiled_header_path))) + break; + + if (!precompiled_header_path[0] || access(precompiled_header_path, F_OK) != -1) { + ret = dcc_x_token_int(out_fd, "PCHR", 0); + break; + } + rs_log_notice("precompiled header %s is missed, try to load", precompiled_header_path); + if ((ret = dcc_mkdir_recursively(precompiled_header_path))) + break; + + precompiled_header_lock = (char *)malloc(strlen(precompiled_header_path) + 1 + 5); + if (!precompiled_header_lock) { + ret = EXIT_OUT_OF_MEMORY; + break; + } + + strcpy(precompiled_header_lock, precompiled_header_path); + strcat(precompiled_header_lock, ".lock"); + + if ((ret = dcc_open_lockfile(precompiled_header_lock, &lock_fd))) + break; + + if ((ret = dcc_lock(lock_fd, 1))) + break; + + if (access(precompiled_header_path, F_OK) != -1) { + rs_log_notice("precompiled header %s was loaded from sibling process", precompiled_header_path); + ret = dcc_x_token_int(out_fd, "PCHR", 0); + break; + } + if ((ret = dcc_x_token_int(out_fd, "PCHR", 1))) + break; + + char *precompiled_header_tmp = precompiled_header_lock; + strcpy(precompiled_header_tmp, precompiled_header_path); + strcat(precompiled_header_tmp, ".tmp"); + + rs_log_notice("loading precompiled header into %s", precompiled_header_tmp); + if ((ret = dcc_r_token_file(in_fd, "PCHF", precompiled_header_tmp, DCC_COMPRESS_LZO1X))) + break; + + ret = dcc_rename(precompiled_header_tmp, precompiled_header_path); + if (!ret) + rs_log_notice("precompiled header %s was successfully loaded", precompiled_header_path); + } while(0); + + free(precompiled_header_path); + free(precompiled_header_lock); + if (lock_fd) { + if (!ret) + return dcc_unlock(lock_fd); + + dcc_unlock(lock_fd); + } + + return ret; +} /** * Read a request, run the compiler, and send a response. @@ -702,6 +777,10 @@ static int dcc_run_job(int in_fd, &tweaked_argv))) goto out_cleanup; + if ((ret = try_load_precompiled_header(in_fd, out_fd))) { + goto out_cleanup; + } + /* The orig_input_tmp and orig_output_tmp values returned by dcc_scan_args() * are aliased with some element of tweaked_argv. We need to copy them, * because the calls to dcc_set_input() and dcc_set_output() below will diff --git a/src/tempfile.c b/src/tempfile.c index 6890efb5..17c62816 100644 --- a/src/tempfile.c +++ b/src/tempfile.c @@ -123,6 +123,41 @@ int dcc_mk_tmpdir(const char *path) return 0; } +/** + * Create the full @path. If it already exists as a directory we succeed. + */ +int dcc_mkdir_recursively(const char *path) +{ + char *copy = 0; + char *p; + int ret; + + copy = strdup(path); + if (copy == NULL) { + return EXIT_OUT_OF_MEMORY; + } + + dcc_truncate_to_dirname(copy); + if (copy[0] == '\0') { + free(copy); + return 0; + } + + for (p = copy; *p != '\0'; ++p) { + if (*p == '/' && p != copy) { + *p = '\0'; + if ((ret = dcc_mkdir(copy))) { + free(copy); + return ret; + } + *p = '/'; + } + } + ret = dcc_mkdir(copy); + free(copy); + return ret; +} + /** * Create the directory @p path. If it already exists as a directory * we succeed. diff --git a/src/util.c b/src/util.c index 27f86202..d008338a 100644 --- a/src/util.c +++ b/src/util.c @@ -564,8 +564,6 @@ int dcc_dup_part(const char **psrc, char **pdst, const char *sep) return 0; } - - int dcc_remove_if_exists(const char *fname) { if (unlink(fname) && errno != ENOENT) { @@ -576,6 +574,16 @@ int dcc_remove_if_exists(const char *fname) return 0; } +int dcc_rename(const char *old_filename, const char *new_filename) +{ + if (rename(old_filename, new_filename)) { + rs_log_warning("failed to rename %s to %s: %s", + old_filename, new_filename, strerror(errno)); + return EXIT_IO_ERROR; + } + return 0; +} + int dcc_which(const char *command, char **out) { char *loc = NULL, *path, *t; diff --git a/src/util.h b/src/util.h index f9a48396..b3c605c1 100644 --- a/src/util.h +++ b/src/util.h @@ -38,6 +38,7 @@ int dcc_getenv_bool(const char *name, int def_value); int set_cloexec_flag (int desc, int value); int dcc_ignore_sigpipe(int val); int dcc_remove_if_exists(const char *fname); +int dcc_rename(const char *old_filename, const char *new_filename); int dcc_trim_path(const char *compiler_name); int dcc_set_path(const char *newpath); char *dcc_abspath(const char *path, int path_len);