bpo-36301: Remove _PyCoreConfig.preconfig (GH-12546)

* Replace _PyCoreConfig.preconfig with 3 new fields in _PyCoreConfig: isolated, use_environment, dev_mode. * Add _PyPreCmdline.dev_mode. * Add _Py_PreInitializeFromPreConfigInPlace().

Author: vstinner

Include/cpython/coreconfig.h

@@ -123,7 +123,9 @@ typedef struct {
 /* --- _PyCoreConfig ---------------------------------------------- */
 
 typedef struct {
- _PyPreConfig preconfig;
+ int isolated;
+ int use_environment;
+ int dev_mode;
 
  /* Install signal handlers? Yes by default. */
  int install_signal_handlers;
@@ -375,7 +377,9 @@ typedef struct {
 #define _PyCoreConfig_INIT \
  (_PyCoreConfig){ \
  _PyCoreConfig_WINDOWS_INIT \
- .preconfig = _PyPreConfig_INIT, \
+ .isolated = -1, \
+ .use_environment = -1, \
+ .dev_mode = -1, \
  .install_signal_handlers = 1, \
  .use_hash_seed = -1, \
  .faulthandler = -1, \

Include/cpython/pylifecycle.h

@@ -16,7 +16,7 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
 
 PyAPI_FUNC(_PyInitError) _Py_PreInitialize(void);
 PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromPreConfig(
- _PyPreConfig *preconfig);
+ const _PyPreConfig *preconfig);
 PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromConfig(
  const _PyCoreConfig *coreconfig);
 

Include/internal/pycore_coreconfig.h

@@ -16,12 +16,14 @@ typedef struct {
  _PyWstrList xoptions; /* "-X value" option */
  int use_environment; /* -E option */
  int isolated; /* -I option */
+ int dev_mode; /* -X dev and PYTHONDEVMODE */
 } _PyPreCmdline;
 
 #define _PyPreCmdline_INIT \
  (_PyPreCmdline){ \
  .use_environment = -1, \
- .isolated = -1}
+ .isolated = -1, \
+ .dev_mode = -1}
 /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */
 
 PyAPI_FUNC(void) _PyPreCmdline_Clear(_PyPreCmdline *cmdline);
@@ -112,7 +114,7 @@ PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config,
  const _PyArgv *args);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_Write(const _PyCoreConfig *config);
+PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config);
 
 /* --- _PyMainInterpreterConfig ----------------------------------- */
 

Include/internal/pycore_pylifecycle.h

@@ -77,6 +77,9 @@ extern void _PyGILState_Fini(void);
 
 PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
 
+PyAPI_FUNC(_PyInitError) _Py_PreInitializeInPlace(
+ _PyPreConfig *config);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_embed.py

@@ -272,12 +272,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  'allocator': None,
  'coerce_c_locale': 0,
  'coerce_c_locale_warn': 0,
- 'dev_mode': 0,
- 'isolated': 0,
- 'use_environment': 1,
  'utf8_mode': 0,
  }
+ COPY_PRE_CONFIG = [
+ 'dev_mode',
+ 'isolated',
+ 'use_environment',
+ ]
+
  DEFAULT_CORE_CONFIG = {
+ 'isolated': 0,
+ 'use_environment': 1,
+ 'dev_mode': 0,
+
  'install_signal_handlers': 1,
  'use_hash_seed': 0,
  'hash_seed': 0,
@@ -363,8 +370,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  '_Py_HasFileSystemDefaultEncodeErrors': 0,
  }
  COPY_GLOBAL_PRE_CONFIG = [
- ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
- ('Py_IsolatedFlag', 'isolated'),
  ('Py_UTF8Mode', 'utf8_mode'),
  ]
  COPY_GLOBAL_CONFIG = [
@@ -376,8 +381,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
  ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
  ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
  ('Py_FrozenFlag', '_frozen'),
+ ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
  ('Py_InspectFlag', 'inspect'),
  ('Py_InteractiveFlag', 'interactive'),
+ ('Py_IsolatedFlag', 'isolated'),
  ('Py_NoSiteFlag', 'site_import', True),
  ('Py_NoUserSiteDirectory', 'user_site_directory', True),
  ('Py_OptimizeFlag', 'optimization_level'),
@@ -415,7 +422,7 @@ def check_main_config(self, config):
  expected['xoptions'] = self.main_xoptions(core_config['xoptions'])
  self.assertEqual(main_config, expected)
 
- def get_expected_config(self, expected, expected_preconfig, env):
+ def get_expected_config(self, expected, env):
  expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
 
  code = textwrap.dedent('''
@@ -443,7 +450,7 @@ def get_expected_config(self, expected, expected_preconfig, env):
  # when test_embed is run from a venv (bpo-35313)
  args = (sys.executable, '-S', '-c', code)
  env = dict(env)
- if not expected_preconfig['isolated']:
+ if not expected['isolated']:
  env['PYTHONCOERCECLOCALE'] = '0'
  env['PYTHONUTF8'] = '0'
  proc = subprocess.run(args, env=env,
@@ -509,7 +516,10 @@ def check_config(self, testname, expected_config, expected_preconfig):
  config = json.loads(out)
 
  expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
- expected_config = self.get_expected_config(expected_config, expected_preconfig, env)
+ expected_config = self.get_expected_config(expected_config, env)
+ for key in self.COPY_PRE_CONFIG:
+ if key not in expected_preconfig:
+ expected_preconfig[key] = expected_config[key]
 
  self.check_core_config(config, expected_config)
  self.check_pre_config(config, expected_preconfig)
@@ -617,35 +627,36 @@ def test_init_env(self):
 
  def test_init_env_dev_mode(self):
  preconfig = dict(self.INIT_ENV_PRECONFIG,
- allocator='debug',
- dev_mode=1)
+ allocator='debug')
  config = dict(self.INIT_ENV_CONFIG,
  dev_mode=1)
  self.check_config("init_env_dev_mode", config, preconfig)
 
- def test_init_env_dev_mode(self):
+ def test_init_env_dev_mode_alloc(self):
  preconfig = dict(self.INIT_ENV_PRECONFIG,
- allocator='malloc',
-  dev_mode=1)
- config = dict(self.INIT_ENV_CONFIG)
+ allocator='malloc')
+ config = dict(self.INIT_ENV_CONFIG,
+  dev_mode=1)
  self.check_config("init_env_dev_mode_alloc", config, preconfig)
 
  def test_init_dev_mode(self):
  preconfig = {
  'allocator': 'debug',
- 'dev_mode': 1,
  }
  config = {
  'faulthandler': 1,
+ 'dev_mode': 1,
  }
  self.check_config("init_dev_mode", config, preconfig)
 
  def test_init_isolated(self):
  preconfig = {
- 'isolated': 1,
- 'use_environment': 0,
+ 'isolated': 0,
+ 'use_environment': 1,
  }
  config = {
+ 'isolated': 1,
+ 'use_environment': 0,
  'user_site_directory': 0,
  }
  self.check_config("init_isolated", config, preconfig)

Modules/main.c

@@ -294,7 +294,7 @@ pymain_init_preconfig(const _PyArgv *args)
  goto done;
  }
 
- err = _Py_PreInitializeFromPreConfig(&config);
+ err = _Py_PreInitializeInPlace(&config);
 
 done:
  _PyPreConfig_Clear(&config);
@@ -311,11 +311,6 @@ pymain_init_coreconfig(_PyCoreConfig *config, const _PyArgv *args,
  return err;
  }
 
- err = _PyCoreConfig_Write(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
- }
-
  return _Py_InitializeCore(interp_p, config);
 }
 
@@ -483,7 +478,7 @@ pymain_header(const _PyCoreConfig *config)
 static void
 pymain_import_readline(const _PyCoreConfig *config)
 {
- if (config->preconfig.isolated) {
+ if (config->isolated) {
  return;
  }
  if (!config->inspect && RUN_CODE(config)) {
@@ -655,7 +650,7 @@ pymain_run_file(_PyCoreConfig *config, PyCompilerFlags *cf)
 static void
 pymain_run_startup(_PyCoreConfig *config, PyCompilerFlags *cf)
 {
- const char *startup = _Py_GetEnv(config->preconfig.use_environment, "PYTHONSTARTUP");
+ const char *startup = _Py_GetEnv(config->use_environment, "PYTHONSTARTUP");
  if (startup == NULL) {
  return;
  }
@@ -735,7 +730,7 @@ pymain_repl(_PyCoreConfig *config, PyCompilerFlags *cf, int *exitcode)
 {
  /* Check this environment variable at the end, to give programs the
  opportunity to set it from Python. */
- if (!Py_InspectFlag && _Py_GetEnv(config->preconfig.use_environment, "PYTHONINSPECT")) {
+ if (!Py_InspectFlag && _Py_GetEnv(config->use_environment, "PYTHONINSPECT")) {
  Py_InspectFlag = 1;
  config->inspect = 1;
  }
@@ -775,7 +770,7 @@ pymain_run_python(PyInterpreterState *interp, int *exitcode)
  goto done;
  }
  }
- else if (!config->preconfig.isolated) {
+ else if (!config->isolated) {
  PyObject *path0 = NULL;
  if (_PyPathConfig_ComputeSysPath0(&config->argv, &path0)) {
  if (path0 == NULL) {

Programs/_freeze_importlib.c

@@ -77,7 +77,7 @@ main(int argc, char *argv[])
  text[text_size] = '\0';
 
  _PyCoreConfig config = _PyCoreConfig_INIT;
- config.preconfig.use_environment = 0;
+ config.use_environment = 0;
  config.user_site_directory = 0;
  config.site_import = 0;
  config.program_name = L"./_freeze_importlib";

Programs/_testembed.c

@@ -397,6 +397,22 @@ static int test_init_global_config(void)
 
 static int test_init_from_config(void)
 {
+ _PyInitError err;
+
+ _PyPreConfig preconfig = _PyPreConfig_INIT;
+
+ putenv("PYTHONMALLOC=malloc_debug");
+ preconfig.allocator = "malloc";
+
+ putenv("PYTHONUTF8=0");
+ Py_UTF8Mode = 0;
+ preconfig.utf8_mode = 1;
+
+ err = _Py_PreInitializeFromPreConfig(&preconfig);
+ if (_Py_INIT_FAILED(err)) {
+ _Py_ExitInitError(err);
+ }
+
  /* Test _Py_InitializeFromConfig() */
  _PyCoreConfig config = _PyCoreConfig_INIT;
  config.install_signal_handlers = 0;
@@ -407,9 +423,6 @@ static int test_init_from_config(void)
  config.use_hash_seed = 1;
  config.hash_seed = 123;
 
- putenv("PYTHONMALLOC=malloc_debug");
- config.preconfig.allocator = "malloc";
-
  /* dev_mode=1 is tested in test_init_dev_mode() */
 
  putenv("PYTHONFAULTHANDLER=");
@@ -430,10 +443,6 @@ static int test_init_from_config(void)
 
  /* FIXME: test coerce_c_locale and coerce_c_locale_warn */
 
- putenv("PYTHONUTF8=0");
- Py_UTF8Mode = 0;
- config.preconfig.utf8_mode = 1;
-
  putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
  config.pycache_prefix = L"conf_pycache_prefix";
 
@@ -521,7 +530,7 @@ static int test_init_from_config(void)
  Py_FrozenFlag = 0;
  config._frozen = 1;
 
- _PyInitError err = _Py_InitializeFromConfig(&config);
+ err = _Py_InitializeFromConfig(&config);
  /* Don't call _PyCoreConfig_Clear() since all strings are static */
  if (_Py_INIT_FAILED(err)) {
  _Py_ExitInitError(err);
@@ -607,20 +616,30 @@ static int test_init_env_dev_mode_alloc(void)
 
 static int test_init_isolated(void)
 {
+ _PyInitError err;
+
+ _PyPreConfig preconfig = _PyPreConfig_INIT;
+
+ /* Set coerce_c_locale and utf8_mode to not depend on the locale */
+ preconfig.coerce_c_locale = 0;
+ preconfig.utf8_mode = 0;
+
+ err = _Py_PreInitializeFromPreConfig(&preconfig);
+ if (_Py_INIT_FAILED(err)) {
+ _Py_ExitInitError(err);
+ }
+
  /* Test _PyCoreConfig.isolated=1 */
  _PyCoreConfig config = _PyCoreConfig_INIT;
 
  Py_IsolatedFlag = 0;
- config.preconfig.isolated = 1;
+ config.isolated = 1;
 
- /* Set coerce_c_locale and utf8_mode to not depend on the locale */
- config.preconfig.coerce_c_locale = 0;
- config.preconfig.utf8_mode = 0;
  /* Use path starting with "./" avoids a search along the PATH */
  config.program_name = L"./_testembed";
 
  test_init_env_dev_mode_putenvs();
- _PyInitError err = _Py_InitializeFromConfig(&config);
+ err = _Py_InitializeFromConfig(&config);
  if (_Py_INIT_FAILED(err)) {
  _Py_ExitInitError(err);
  }
@@ -635,7 +654,7 @@ static int test_init_dev_mode(void)
  _PyCoreConfig config = _PyCoreConfig_INIT;
  putenv("PYTHONFAULTHANDLER=");
  putenv("PYTHONMALLOC=");
- config.preconfig.dev_mode = 1;
+ config.dev_mode = 1;
  config.program_name = L"./_testembed";
  _PyInitError err = _Py_InitializeFromConfig(&config);
  if (_Py_INIT_FAILED(err)) {