diff --git a/configure.ac b/configure.ac index 7bbe190f6f..02fcf29a1e 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,8 @@ AS_IF([test "x$enable_criu" != "xno"], [ AC_MSG_NOTICE([CRIU version doesn't support for pre-dumping])]) PKG_CHECK_MODULES([CRIU_NETWORK_LOCK_SKIP], [criu >= 3.19], [have_criu_network_lock_skip="yes"], [have_criu_network_lock_skip="no" AC_MSG_NOTICE([CRIU version doesn't support CRIU_NETWORK_LOCK_SKIP])]) + PKG_CHECK_MODULES([CRIU_CONFIG_FILE], [criu >= 4.2], [have_criu_config_file="yes"], [have_criu_config_file="no" + AC_MSG_NOTICE([libcriu version doesn't support setting RPC config file])]) AS_IF([test "$have_criu" = "yes"], [ AC_DEFINE([HAVE_CRIU], 1, [Define if CRIU is available]) ]) @@ -278,6 +280,9 @@ AS_IF([test "x$enable_criu" != "xno"], [ AS_IF([test "$have_criu_network_lock_skip" = "yes"], [ AC_DEFINE([CRIU_NETWORK_LOCK_SKIP_SUPPORT], 1, [Define if CRIU_NETWORK_LOCK_SKIP is available]) ]) + AS_IF([test "$have_criu_config_file" = "yes"], [ + AC_DEFINE([CRIU_CONFIG_FILE], 1, [Define if CRIU_CONFIG_FILE is available]) + ]) ], [AC_MSG_NOTICE([CRIU support disabled per user request])]) diff --git a/crun.1 b/crun.1 index 0ca94941d0..79821a1dd5 100644 --- a/crun.1 +++ b/crun.1 @@ -649,6 +649,17 @@ workload natively. Accepts a \fB\&.wasm\fR binary as input and if \fB\&.wat\fR i provided it will be automatically compiled into a wasm module. Stdout of wasm module is relayed back via crun. +.SH \fBorg.criu.config=FILE\fR +This annotation allows specifying a CRIU RPC configuration file. +If not provided, crun will look for \fB/etc/criu/crun.conf\fR\&. If +neither the annotation nor \fBcrun.conf\fR exists, \fB/etc/criu/runc.conf\fR +is used for compatibility with runc. This functionality requires +CRIU version 4.2 or newer. The options specified in the configuration +file override the default values of CRIU options specified by crun. +For example, specifying \fBtcp-established\fR or \fBtcp-close\fR can be used +to checkpoint/restore containers with TCP established connections, +and \fBlog-file=\fR to set a custom CRIU logs file. + .SH tmpcopyup mount options If the \fBtmpcopyup\fR option is specified for a tmpfs, then the path that is shadowed by the tmpfs mount is recursively copied up to the tmpfs @@ -848,8 +859,7 @@ l l l l . \fBOCI (x)\fP \fBcgroup 2 value (y)\fP \fBconversion\fP \fBcomment\fP shares cpu.weight T{ y=10^((log2(x)^2 + 125 * log2(x)) / 612.0 - 7.0 / 34.0) -T} - T{ +T} T{ convert from [2-262144] to [1-10000] T} period cpu.max y = x T{ diff --git a/crun.1.md b/crun.1.md index d1e136d9a4..7bdb1dbbe4 100644 --- a/crun.1.md +++ b/crun.1.md @@ -557,6 +557,18 @@ workload natively. Accepts a `.wasm` binary as input and if `.wat` is provided it will be automatically compiled into a wasm module. Stdout of wasm module is relayed back via crun. +## `org.criu.config=FILE` + +This annotation allows specifying a CRIU RPC configuration file. +If not provided, crun will look for `/etc/criu/crun.conf`. If +neither the annotation nor `crun.conf` exists, `/etc/criu/runc.conf` +is used for compatibility with runc. This functionality requires +CRIU version 4.2 or newer. The options specified in the configuration +file override the default values of CRIU options specified by crun. +For example, specifying `tcp-established` or `tcp-close` can be used +to checkpoint/restore containers with TCP established connections, +and `log-file=` to set a custom CRIU logs file. + ## tmpcopyup mount options If the `tmpcopyup` option is specified for a tmpfs, then the path that diff --git a/src/libcrun/container.c b/src/libcrun/container.c index 1b2ad3aaf8..299709b255 100644 --- a/src/libcrun/container.c +++ b/src/libcrun/container.c @@ -382,6 +382,7 @@ static char *potentially_unsafe_annotations[] = { "module.wasm.image/variant", "io.kubernetes.cri.container-type", "run.oci.", + "org.criu.", }; #define SYNC_SOCKET_MESSAGE_LEN(x, l) (offsetof (struct sync_socket_message_s, message) + l) diff --git a/src/libcrun/criu.c b/src/libcrun/criu.c index 45c1cce81b..abe8153029 100644 --- a/src/libcrun/criu.c +++ b/src/libcrun/criu.c @@ -43,6 +43,8 @@ # define CRIU_CHECKPOINT_LOG_FILE "dump.log" # define CRIU_RESTORE_LOG_FILE "restore.log" # define DESCRIPTORS_FILENAME "descriptors.json" +# define CRIU_RUNC_CONFIG_FILE "/etc/criu/runc.conf" +# define CRIU_CRUN_CONFIG_FILE "/etc/criu/crun.conf" # define CRIU_EXT_NETNS "extRootNetNS" # define CRIU_EXT_PIDNS "extRootPidNS" @@ -99,6 +101,7 @@ struct libcriu_wrapper_s void (*criu_set_work_dir_fd) (int fd); int (*criu_set_lsm_profile) (const char *name); int (*criu_set_lsm_mount_context) (const char *name); + int (*criu_set_config_file) (const char *path); }; static struct libcriu_wrapper_s *libcriu_wrapper; @@ -194,6 +197,9 @@ load_wrapper (struct libcriu_wrapper_s **wrapper_out, libcrun_error_t *err) LOAD_CRIU_FUNCTION (criu_set_work_dir_fd, false); LOAD_CRIU_FUNCTION (criu_set_lsm_profile, false); LOAD_CRIU_FUNCTION (criu_set_lsm_mount_context, false); +# if ! defined STATIC || defined CRIU_CONFIG_FILE + LOAD_CRIU_FUNCTION (criu_set_config_file, true); +# endif libcriu_wrapper = *wrapper_out = wrapper; wrapper = NULL; @@ -460,6 +466,38 @@ checkpoint_cgroup_v1_mount (runtime_spec_schema_config_schema *def, libcrun_erro return 0; } +static int +handle_criu_config_file (libcrun_container_t *container, libcrun_error_t *err) +{ + int ret; + const char *criu_config_annotation; + const char *config_file = CRIU_RUNC_CONFIG_FILE; + + criu_config_annotation = find_annotation (container, "org.criu.config"); + + /* Ignore missing criu_set_config_file() API for compatibility with older CRIU versions, + * and show an error only if config file is explicitly set with annotation. + */ + if (libcriu_wrapper->criu_set_config_file == NULL) + { + if (criu_config_annotation) + return crun_make_error (err, 0, "libcriu RPC config files supported in CRIU >= 4.2"); + + return 0; + } + + if (criu_config_annotation) + config_file = criu_config_annotation; + else if (access (CRIU_CRUN_CONFIG_FILE, F_OK) == 0) + config_file = CRIU_CRUN_CONFIG_FILE; + + ret = libcriu_wrapper->criu_set_config_file (config_file); + if (UNLIKELY (ret < 0)) + return crun_make_error (err, 0, "failed to set CRIU config file"); + + return 0; +} + int libcrun_container_checkpoint_linux_criu (libcrun_container_status_t *status, libcrun_container_t *container, libcrun_checkpoint_restore_t *cr_options, libcrun_error_t *err) @@ -522,6 +560,11 @@ libcrun_container_checkpoint_linux_criu (libcrun_container_status_t *status, lib /* Set up logging. */ libcriu_wrapper->criu_set_log_level (4); libcriu_wrapper->criu_set_log_file (CRIU_CHECKPOINT_LOG_FILE); + + /* Set up CRIU config file */ + if (UNLIKELY (handle_criu_config_file (container, err))) + return -1; + /* Setting the pid early as we can skip a lot of checkpoint setup if * we just do a pre-dump. The PID needs to be set always. Do it here. * The main process of the container is the process CRIU will checkpoint @@ -1098,6 +1141,10 @@ libcrun_container_restore_linux_criu (libcrun_container_status_t *status, libcru # endif } + /* Set up CRIU config file */ + if (UNLIKELY (handle_criu_config_file (container, err))) + return -1; + /* Tell CRIU if cgroup v1 needs to be handled. */ ret = restore_cgroup_v1_mount (def, err); if (UNLIKELY (ret < 0)) diff --git a/tests/test_checkpoint_restore.py b/tests/test_checkpoint_restore.py index f4280c1814..646d52e6ad 100755 --- a/tests/test_checkpoint_restore.py +++ b/tests/test_checkpoint_restore.py @@ -19,6 +19,8 @@ import json import os import subprocess +import errno +import tempfile from tests_utils import * criu_version = 0 @@ -82,7 +84,7 @@ def _get_cmdline(cid, tests_root): return "" -def run_cr_test(conf): +def run_cr_test(conf, before_checkpoint_cb=None, before_restore_cb=None): cid = None cr_dir = os.path.join(get_tests_root(), 'checkpoint') work_dir = 'work-dir' @@ -98,6 +100,9 @@ def run_cr_test(conf): if first_cmdline == "": return -1 + if before_checkpoint_cb is not None: + before_checkpoint_cb() + run_crun_command([ "checkpoint", "--image-path=%s" % cr_dir, @@ -110,6 +115,9 @@ def run_cr_test(conf): cid.split('-')[1] ) + if before_restore_cb is not None: + before_restore_cb() + run_crun_command([ "restore", "-d", @@ -263,10 +271,99 @@ def test_cr_with_ext_ns(): return run_cr_test(conf) +def _remove_file(filename): + try: + os.remove(filename) + except OSError as e: + # ignore "no such file" and raise other exceptions + if e.errno != errno.ENOENT: + raise + + +def _clean_up_criu_configs(): + for conf_file in ["crun.conf", "runc.conf", "annotation.conf"]: + _remove_file(os.path.join("/etc/criu", conf_file)) + + +def _create_criu_config(file_name, content): + config_dir = "/etc/criu/" + os.makedirs(config_dir, 0o755, exist_ok=True) + with open(os.path.join(config_dir, f"{file_name}.conf"), "w") as f: + print(content, file=f) + + +def _run_cr_test_with_config(config_name, log_names, extra_configs=None, annotations=None): + """ + Helper to run CRIU tests with a configuration file. + + :param config_name: The main config to create before checkpoint/restore. + :param log_names: Tuple of (dump_log_name, restore_log_name) for the test. + :param extra_configs: Optional dict of extra config_name -> content to create before test. + :param annotations: Optional dict of annotations to set in the config. + :return: 0 on success, -1 on failure. + """ + conf = base_config() + conf['process']['args'] = ['/init', 'pause'] + + with tempfile.TemporaryDirectory() as tmp: + dump_log_path = os.path.join(tmp, log_names[0]) + restore_log_path = os.path.join(tmp, log_names[1]) + + if extra_configs: + for name, content in extra_configs.items(): + _create_criu_config(name, content) + + if annotations: + conf['annotations'] = annotations + + ret = run_cr_test( + conf, + before_checkpoint_cb=lambda: _create_criu_config(config_name, f"log-file={dump_log_path}"), + before_restore_cb=lambda: _create_criu_config(config_name, f"log-file={restore_log_path}") + ) + _clean_up_criu_configs() + + if ret != 0: + return ret + + for path in [dump_log_path, restore_log_path]: + if not os.path.isfile(path) or os.path.getsize(path) == 0: + return -1 + return 0 + + +def test_cr_with_runc_config(): + if is_rootless() or 'CRIU' not in get_crun_feature_string(): + return 77 + return _run_cr_test_with_config("runc", ("runc-dump.log", "runc-restore.log")) + + +def test_cr_with_crun_config(): + if is_rootless() or 'CRIU' not in get_crun_feature_string(): + return 77 + # runc.conf should be ignored by crun + extra = {"runc": "log-file=test.log"} + return _run_cr_test_with_config("crun", ("crun-dump.log", "crun-restore.log"), extra_configs=extra) + + +def test_cr_with_annotation_config(): + if is_rootless() or 'CRIU' not in get_crun_feature_string(): + return 77 + # Create annotation config file + annotations = {"org.criu.config": "/etc/criu/annotation.conf"} + _create_criu_config("annotation", f"log-file=annotation.log") + # The following config files should be ignored by crun + extra = {"runc": "log-file=test-runc.log", "crun": "log-file=test-crun.log"} + return _run_cr_test_with_config("annotation", ("dump.log", "restore.log"), extra_configs=extra, annotations=annotations) + + all_tests = { "checkpoint-restore": test_cr, "checkpoint-restore-ext-ns": test_cr_with_ext_ns, "checkpoint-restore-pre-dump": test_cr_pre_dump, + "checkpoint-restore-with-runc-config": test_cr_with_runc_config, + "checkpoint-restore-with-crun-config": test_cr_with_crun_config, + "checkpoint-restore-with-annotation-config": test_cr_with_annotation_config, } if __name__ == "__main__": diff --git a/tests/test_oci_features.py b/tests/test_oci_features.py index 4c09db6b20..55c49e7d6f 100755 --- a/tests/test_oci_features.py +++ b/tests/test_oci_features.py @@ -174,6 +174,7 @@ def test_crun_features(): "module.wasm.image/variant", "io.kubernetes.cri.container-type", "run.oci.", + "org.criu.", ] }