builder: Add support for patching mtime in python bytecode headers

This makes them work at runtime (as the mtime will then be 0) and
makes builds more repeatable.

builder/builder-module.c

@@ -49,6 +49,7 @@ struct BuilderModule
   gboolean        rm_configure;
   gboolean        no_autogen;
   gboolean        no_parallel_make;
+  gboolean        no_python_timestamp_fix;
   gboolean        cmake;
   gboolean        builddir;
   BuilderOptions *build_options;
@@ -76,6 +77,7 @@ enum {
   PROP_DISABLED,
   PROP_NO_AUTOGEN,
   PROP_NO_PARALLEL_MAKE,
+  PROP_NO_PYTHON_TIMESTAMP_FIX,
   PROP_CMAKE,
   PROP_BUILDDIR,
   PROP_CONFIG_OPTS,
@@ -146,6 +148,10 @@ builder_module_get_property (GObject    *object,
       g_value_set_boolean (value, self->no_parallel_make);
       break;
 
+    case PROP_NO_PYTHON_TIMESTAMP_FIX:
+      g_value_set_boolean (value, self->no_python_timestamp_fix);
+      break;
+
     case PROP_CMAKE:
       g_value_set_boolean (value, self->cmake);
       break;
@@ -232,6 +238,10 @@ builder_module_set_property (GObject      *object,
       self->no_parallel_make = g_value_get_boolean (value);
       break;
 
+    case PROP_NO_PYTHON_TIMESTAMP_FIX:
+      self->no_python_timestamp_fix = g_value_get_boolean (value);
+      break;
+
     case PROP_CMAKE:
       self->cmake = g_value_get_boolean (value);
       break;
@@ -342,6 +352,13 @@ builder_module_class_init (BuilderModuleClass *klass)
                                                          "",
                                                          FALSE,
                                                          G_PARAM_READWRITE));
+  g_object_class_install_property (object_class,
+                                   PROP_NO_PYTHON_TIMESTAMP_FIX,
+                                   g_param_spec_boolean ("no-python-timestamp-fix",
+                                                         "",
+                                                         "",
+                                                         FALSE,
+                                                         G_PARAM_READWRITE));
   g_object_class_install_property (object_class,
                                    PROP_CMAKE,
                                    g_param_spec_boolean ("cmake",
@@ -868,6 +885,124 @@ builder_module_handle_debuginfo (BuilderModule  *self,
   return TRUE;
 }
 
+static gboolean
+fixup_python_timestamp (int dfd,
+                        const char *rel_path,
+                        const char *full_path,
+                        GCancellable  *cancellable,
+                        GError       **error)
+{
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+
+  glnx_dirfd_iterator_init_at (dfd, rel_path, FALSE, &dfd_iter, NULL);
+
+  while (TRUE)
+    {
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, NULL) || dent == NULL)
+        break;
+
+      if (dent->d_type == DT_DIR)
+        {
+          g_autofree char *child_full_path = g_build_filename (full_path, dent->d_name, NULL);
+          if (!fixup_python_timestamp (dfd_iter.fd, dent->d_name, child_full_path,
+                                       cancellable, error))
+            return FALSE;
+        }
+      else if (dent->d_type == DT_REG &&
+               dfd != AT_FDCWD &&
+               (g_str_has_suffix (dent->d_name, ".pyc") ||
+                g_str_has_suffix (dent->d_name, ".pyo")))
+        {
+          glnx_fd_close int fd = -1;
+          guint8 buffer[8];
+          ssize_t res;
+          guint32 mtime;
+          g_autofree char *new_path = NULL;
+          struct stat stbuf;
+
+          fd = openat (dfd_iter.fd, dent->d_name, O_RDWR | O_CLOEXEC | O_NOFOLLOW);
+          if (fd == -1)
+            {
+              g_warning ("Can't open %s", dent->d_name);
+              continue;
+            }
+
+          res = read (fd, buffer, 8);
+          if (res != 8)
+            {
+              g_warning ("Short read for %s", dent->d_name);
+              continue;
+            }
+
+          if (buffer[2] != 0x0d || buffer[3] != 0x0a)
+            {
+              g_debug ("Not matching python magic: %s", dent->d_name);
+              continue;
+            }
+
+          mtime =
+            (buffer[4] << 8*0) |
+            (buffer[5] << 8*1) |
+            (buffer[6] << 8*2) |
+            (buffer[7] << 8*3);
+
+          if (mtime == 0)
+            continue; /* Already zero, ignore */
+
+          if (strcmp (rel_path, "__pycache__") == 0)
+            {
+              /* Python3 */
+              g_autofree char *base = g_strdup (dent->d_name);
+              char *dot;
+
+              dot = strrchr (base, '.');
+              if (dot == NULL)
+                continue;
+              *dot = 0;
+
+              dot = strrchr (base, '.');
+              if (dot == NULL)
+                continue;
+              *dot = 0;
+
+              new_path = g_strconcat ("../", base, ".py", NULL);
+            }
+          else
+            {
+              /* Python2 */
+              new_path = g_strndup (dent->d_name, strlen (dent->d_name) - 1);
+            }
+
+          if (fstatat (dfd_iter.fd, new_path, &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
+            continue;
+
+          if (stbuf.st_mtime != mtime)
+            continue;
+
+          buffer[4] = buffer[5] = buffer[6] = buffer[7] = 0;
+
+          res = pwrite (fd, buffer, 8, 0);
+          if (res != 8)
+            {
+              glnx_set_error_from_errno (error);
+              return FALSE;
+            }
+
+          {
+            g_autofree char *child_full_path = g_build_filename (full_path, dent->d_name, NULL);
+            g_print ("Fixed up header mtime for %s\n", child_full_path);
+          }
+
+          /* The mtime will be zeroed on cache commit. We don't want to do that now, because multiple
+             files could reference one .py file and we need the mtimes to match for them all */
+        }
+    }
+
+  return TRUE;
+}
+
 gboolean
 builder_module_build (BuilderModule  *self,
                       BuilderCache   *cache,
@@ -1171,6 +1306,15 @@ builder_module_build (BuilderModule  *self,
         }
     }
 
+  if (!self->no_python_timestamp_fix)
+    {
+      if (!fixup_python_timestamp (AT_FDCWD,
+                                   gs_file_get_path_cached (app_dir), "/",
+                                   NULL,
+                                   error))
+        return FALSE;
+    }
+
   if (!builder_module_handle_debuginfo (self, app_dir, cache, context, error))
     return FALSE;
 
@@ -1233,6 +1377,7 @@ builder_module_checksum (BuilderModule  *self,
   builder_cache_checksum_boolean (cache, self->no_autogen);
   builder_cache_checksum_boolean (cache, self->disabled);
   builder_cache_checksum_boolean (cache, self->no_parallel_make);
+  builder_cache_checksum_boolean (cache, self->no_python_timestamp_fix);
   builder_cache_checksum_boolean (cache, self->cmake);
   builder_cache_checksum_boolean (cache, self->builddir);
 

doc/flatpak-builder.xml

@@ -316,6 +316,10 @@
                     <term><option>no-parallel-make</option> (boolean)</term>
                     <listitem><para>Don't call make with arguments to build in parallel</para></listitem>
                 </varlistentry>
+                <varlistentry>
+                    <term><option>no-python-timestamp-fix</option> (boolean)</term>
+                    <listitem><para>Don't fix up the *.py[oc] header timestamps for ostree use.</para></listitem>
+                </varlistentry>
                 <varlistentry>
                     <term><option>cmake</option> (boolean)</term>
                     <listitem><para>Use cmake instead of configure</para></listitem>