From bd66da0dad92c0c0432cf35fcf137d4e5e35cf32 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 8 Jun 2016 13:28:27 +0200 Subject: [PATCH] builder: Add support for patching mtime in python bytecode headers This makes them work at runtime (as the mtime will then be 0) and makes builds more repeatable. --- builder/builder-module.c | 145 +++++++++++++++++++++++++++++++++++++++ doc/flatpak-builder.xml | 4 ++ 2 files changed, 149 insertions(+) diff --git a/builder/builder-module.c b/builder/builder-module.c index 2b81e1ff..4c676b10 100644 --- a/builder/builder-module.c +++ b/builder/builder-module.c @@ -49,6 +49,7 @@ struct BuilderModule gboolean rm_configure; gboolean no_autogen; gboolean no_parallel_make; + gboolean no_python_timestamp_fix; gboolean cmake; gboolean builddir; BuilderOptions *build_options; @@ -76,6 +77,7 @@ enum { PROP_DISABLED, PROP_NO_AUTOGEN, PROP_NO_PARALLEL_MAKE, + PROP_NO_PYTHON_TIMESTAMP_FIX, PROP_CMAKE, PROP_BUILDDIR, PROP_CONFIG_OPTS, @@ -146,6 +148,10 @@ builder_module_get_property (GObject *object, g_value_set_boolean (value, self->no_parallel_make); break; + case PROP_NO_PYTHON_TIMESTAMP_FIX: + g_value_set_boolean (value, self->no_python_timestamp_fix); + break; + case PROP_CMAKE: g_value_set_boolean (value, self->cmake); break; @@ -232,6 +238,10 @@ builder_module_set_property (GObject *object, self->no_parallel_make = g_value_get_boolean (value); break; + case PROP_NO_PYTHON_TIMESTAMP_FIX: + self->no_python_timestamp_fix = g_value_get_boolean (value); + break; + case PROP_CMAKE: self->cmake = g_value_get_boolean (value); break; @@ -342,6 +352,13 @@ builder_module_class_init (BuilderModuleClass *klass) "", FALSE, G_PARAM_READWRITE)); + g_object_class_install_property (object_class, + PROP_NO_PYTHON_TIMESTAMP_FIX, + g_param_spec_boolean ("no-python-timestamp-fix", + "", + "", + FALSE, + G_PARAM_READWRITE)); g_object_class_install_property (object_class, PROP_CMAKE, g_param_spec_boolean ("cmake", @@ -868,6 +885,124 @@ builder_module_handle_debuginfo (BuilderModule *self, return TRUE; } +static gboolean +fixup_python_timestamp (int dfd, + const char *rel_path, + const char *full_path, + GCancellable *cancellable, + GError **error) +{ + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + + glnx_dirfd_iterator_init_at (dfd, rel_path, FALSE, &dfd_iter, NULL); + + while (TRUE) + { + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, NULL) || dent == NULL) + break; + + if (dent->d_type == DT_DIR) + { + g_autofree char *child_full_path = g_build_filename (full_path, dent->d_name, NULL); + if (!fixup_python_timestamp (dfd_iter.fd, dent->d_name, child_full_path, + cancellable, error)) + return FALSE; + } + else if (dent->d_type == DT_REG && + dfd != AT_FDCWD && + (g_str_has_suffix (dent->d_name, ".pyc") || + g_str_has_suffix (dent->d_name, ".pyo"))) + { + glnx_fd_close int fd = -1; + guint8 buffer[8]; + ssize_t res; + guint32 mtime; + g_autofree char *new_path = NULL; + struct stat stbuf; + + fd = openat (dfd_iter.fd, dent->d_name, O_RDWR | O_CLOEXEC | O_NOFOLLOW); + if (fd == -1) + { + g_warning ("Can't open %s", dent->d_name); + continue; + } + + res = read (fd, buffer, 8); + if (res != 8) + { + g_warning ("Short read for %s", dent->d_name); + continue; + } + + if (buffer[2] != 0x0d || buffer[3] != 0x0a) + { + g_debug ("Not matching python magic: %s", dent->d_name); + continue; + } + + mtime = + (buffer[4] << 8*0) | + (buffer[5] << 8*1) | + (buffer[6] << 8*2) | + (buffer[7] << 8*3); + + if (mtime == 0) + continue; /* Already zero, ignore */ + + if (strcmp (rel_path, "__pycache__") == 0) + { + /* Python3 */ + g_autofree char *base = g_strdup (dent->d_name); + char *dot; + + dot = strrchr (base, '.'); + if (dot == NULL) + continue; + *dot = 0; + + dot = strrchr (base, '.'); + if (dot == NULL) + continue; + *dot = 0; + + new_path = g_strconcat ("../", base, ".py", NULL); + } + else + { + /* Python2 */ + new_path = g_strndup (dent->d_name, strlen (dent->d_name) - 1); + } + + if (fstatat (dfd_iter.fd, new_path, &stbuf, AT_SYMLINK_NOFOLLOW) != 0) + continue; + + if (stbuf.st_mtime != mtime) + continue; + + buffer[4] = buffer[5] = buffer[6] = buffer[7] = 0; + + res = pwrite (fd, buffer, 8, 0); + if (res != 8) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + { + g_autofree char *child_full_path = g_build_filename (full_path, dent->d_name, NULL); + g_print ("Fixed up header mtime for %s\n", child_full_path); + } + + /* The mtime will be zeroed on cache commit. We don't want to do that now, because multiple + files could reference one .py file and we need the mtimes to match for them all */ + } + } + + return TRUE; +} + gboolean builder_module_build (BuilderModule *self, BuilderCache *cache, @@ -1171,6 +1306,15 @@ builder_module_build (BuilderModule *self, } } + if (!self->no_python_timestamp_fix) + { + if (!fixup_python_timestamp (AT_FDCWD, + gs_file_get_path_cached (app_dir), "/", + NULL, + error)) + return FALSE; + } + if (!builder_module_handle_debuginfo (self, app_dir, cache, context, error)) return FALSE; @@ -1233,6 +1377,7 @@ builder_module_checksum (BuilderModule *self, builder_cache_checksum_boolean (cache, self->no_autogen); builder_cache_checksum_boolean (cache, self->disabled); builder_cache_checksum_boolean (cache, self->no_parallel_make); + builder_cache_checksum_boolean (cache, self->no_python_timestamp_fix); builder_cache_checksum_boolean (cache, self->cmake); builder_cache_checksum_boolean (cache, self->builddir); diff --git a/doc/flatpak-builder.xml b/doc/flatpak-builder.xml index 9e397eb5..9a0575d4 100644 --- a/doc/flatpak-builder.xml +++ b/doc/flatpak-builder.xml @@ -316,6 +316,10 @@ (boolean) Don't call make with arguments to build in parallel + + (boolean) + Don't fix up the *.py[oc] header timestamps for ostree use. + (boolean) Use cmake instead of configure