diff --git a/cmd/podman/shared/create.go b/cmd/podman/shared/create.go
index 99538b3dc8..5b244699c6 100644
--- a/cmd/podman/shared/create.go
+++ b/cmd/podman/shared/create.go
@@ -701,9 +701,6 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod.
 		Sysctl:         sysctl,
 	}
 
-	if err := secConfig.SetLabelOpts(runtime, pid, ipc); err != nil {
-		return nil, err
-	}
 	if err := secConfig.SetSecurityOpts(runtime, c.StringArray("security-opt")); err != nil {
 		return nil, err
 	}
diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go
index b2a152a2d0..21b6bc3b37 100644
--- a/pkg/spec/spec.go
+++ b/pkg/spec/spec.go
@@ -241,14 +241,6 @@ func (config *CreateConfig) createConfigToOCISpec(runtime *libpod.Runtime, userM
 	}
 
 	// SECURITY OPTS
-	g.SetProcessNoNewPrivileges(config.Security.NoNewPrivs)
-
-	if !config.Security.Privileged {
-		g.SetProcessApparmorProfile(config.Security.ApparmorProfile)
-	}
-
-	blockAccessToKernelFilesystems(config, &g)
-
 	var runtimeConfig *libpodconfig.Config
 
 	if runtime != nil {
@@ -258,6 +250,26 @@ func (config *CreateConfig) createConfigToOCISpec(runtime *libpod.Runtime, userM
 		}
 	}
 
+	g.SetProcessNoNewPrivileges(config.Security.NoNewPrivs)
+
+	if !config.Security.Privileged {
+		g.SetProcessApparmorProfile(config.Security.ApparmorProfile)
+	}
+
+	// Unless already set via the CLI, check if we need to disable process
+	// labels or set the defaults.
+	if len(config.Security.LabelOpts) == 0 && runtimeConfig != nil {
+		if !runtimeConfig.EnableLabeling {
+			// Disabled in the config.
+			config.Security.LabelOpts = append(config.Security.LabelOpts, "disable")
+		} else if err := config.Security.SetLabelOpts(runtime, &config.Pid, &config.Ipc); err != nil {
+			// Defaults!
+			return nil, err
+		}
+	}
+
+	blockAccessToKernelFilesystems(config, &g)
+
 	// RESOURCES - PIDS
 	if config.Resources.PidsLimit > 0 {
 		// if running on rootless on a cgroupv1 machine or using the cgroupfs manager, pids