Skip to content

bpo-36945: Add _PyPreConfig.configure_locale #13368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
May 17, 2019
Merged

bpo-36945: Add _PyPreConfig.configure_locale #13368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59a005d
bpo-36945: Add _PyPreConfig.configure_locale
vstinner May 16, 2019
582a5b9
Fix _PyPreConfig_Write()
vstinner May 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Include/cpython/coreconfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ typedef struct {
set to !Py_IgnoreEnvironmentFlag. */
int use_environment;

/* Set the LC_CTYPE locale to the user preferred locale? If equals to 0,
set coerce_c_locale and coerce_c_locale_warn to 0. */
int configure_locale;

/* Coerce the LC_CTYPE locale if it's equal to "C"? (PEP 538)

Set to 0 by PYTHONCOERCECLOCALE=0. Set to 1 by PYTHONCOERCECLOCALE=1.
Expand Down Expand Up @@ -147,6 +151,7 @@ typedef struct {
._config_version = _Py_CONFIG_VERSION, \
.isolated = -1, \
.use_environment = -1, \
.configure_locale = 1, \
.utf8_mode = -2, \
.dev_mode = -1, \
.allocator = PYMEM_ALLOCATOR_NOT_SET}
Expand Down
34 changes: 30 additions & 4 deletions Lib/test/test_embed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -272,10 +272,15 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
maxDiff = 4096
UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape')

# Mark config which should be get by get_default_config()
# Marker to read the default configuration: get_default_config()
GET_DEFAULT_CONFIG = object()

# Marker to ignore a configuration parameter
IGNORE_CONFIG = object()

DEFAULT_PRE_CONFIG = {
'allocator': PYMEM_ALLOCATOR_NOT_SET,
'configure_locale': 1,
'coerce_c_locale': 0,
'coerce_c_locale_warn': 0,
'utf8_mode': 0,
Expand Down Expand Up @@ -405,7 +410,7 @@ def main_xoptions(self, xoptions_list):
xoptions[opt] = True
return xoptions

def get_expected_config(self, expected, env, add_path=None):
def get_expected_config(self, expected_preconfig, expected, env, add_path=None):
expected = dict(self.DEFAULT_CORE_CONFIG, **expected)

code = textwrap.dedent('''
Expand Down Expand Up @@ -471,6 +476,14 @@ def get_expected_config(self, expected, env, add_path=None):

if add_path is not None:
expected['module_search_paths'].append(add_path)

if not expected_preconfig['configure_locale']:
# there is no easy way to get the locale encoding before
# setlocale(LC_CTYPE, "") is called: don't test encodings
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG

return expected

def check_pre_config(self, config, expected):
Expand All @@ -480,6 +493,10 @@ def check_pre_config(self, config, expected):

def check_core_config(self, config, expected):
core_config = dict(config['core_config'])
for key, value in list(expected.items()):
if value is self.IGNORE_CONFIG:
del core_config[key]
del expected[key]
self.assertEqual(core_config, expected)

def check_global_config(self, config):
Expand Down Expand Up @@ -517,7 +534,7 @@ def check_config(self, testname, expected_config, expected_preconfig,
env['PYTHONUTF8'] = '0'

expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
expected_config = self.get_expected_config(expected_config, env, add_path)
expected_config = self.get_expected_config(expected_preconfig, expected_config, env, add_path)
for key in self.COPY_PRE_CONFIG:
if key not in expected_preconfig:
expected_preconfig[key] = expected_config[key]
Expand Down Expand Up @@ -692,7 +709,9 @@ def test_preinit_isolated2(self):
self.check_config("preinit_isolated2", config, preconfig)

def test_init_isolated_config(self):
preconfig = {}
preconfig = {
'configure_locale': 0,
}
config = {
'isolated': 1,
'use_environment': 0,
Expand All @@ -710,6 +729,13 @@ def test_init_python_config(self):
}
self.check_config("init_python_config", config, preconfig)

def test_init_dont_configure_locale(self):
# _PyPreConfig.configure_locale=0
preconfig = {
'configure_locale': 0,
}
self.check_config("init_dont_configure_locale", {}, preconfig)

def test_init_read_set(self):
preconfig = {}
core_config = {
Expand Down
28 changes: 28 additions & 0 deletions Programs/_testembed.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,33 @@ static int init_python_config(void)
}


static int init_dont_configure_locale(void)
{
_PyInitError err;

_PyPreConfig preconfig = _PyPreConfig_INIT;
preconfig.configure_locale = 0;
preconfig.coerce_c_locale = 1;
preconfig.coerce_c_locale_warn = 1;

err = _Py_PreInitialize(&preconfig);
if (_Py_INIT_FAILED(err)) {
_Py_ExitInitError(err);
}

_PyCoreConfig config = _PyCoreConfig_INIT;
config.program_name = L"./_testembed";
err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
_Py_ExitInitError(err);
}

dump_config();
Py_Finalize();
return 0;
}


static int init_dev_mode(void)
{
_PyCoreConfig config;
Expand Down Expand Up @@ -966,6 +993,7 @@ static struct TestCase TestCases[] = {
{ "init_env", test_init_env },
{ "init_env_dev_mode", test_init_env_dev_mode },
{ "init_env_dev_mode_alloc", test_init_env_dev_mode_alloc },
{ "init_dont_configure_locale", init_dont_configure_locale },
{ "init_dev_mode", init_dev_mode },
{ "init_isolated_flag", init_isolated_flag },
{ "init_isolated_config", init_isolated_config },
Expand Down
25 changes: 19 additions & 6 deletions Python/preconfig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config)
{
_PyPreConfig_Init(config);

config->configure_locale = 0;
config->isolated = 1;
config->use_environment = 0;
#ifdef MS_WINDOWS
Expand All @@ -312,6 +313,7 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)

COPY_ATTR(isolated);
COPY_ATTR(use_environment);
COPY_ATTR(configure_locale);
COPY_ATTR(dev_mode);
COPY_ATTR(coerce_c_locale);
COPY_ATTR(coerce_c_locale_warn);
Expand Down Expand Up @@ -360,6 +362,7 @@ _PyPreConfig_AsDict(const _PyPreConfig *config)

SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
SET_ITEM_INT(configure_locale);
SET_ITEM_INT(coerce_c_locale);
SET_ITEM_INT(coerce_c_locale_warn);
SET_ITEM_INT(utf8_mode);
Expand Down Expand Up @@ -603,6 +606,12 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
static void
preconfig_init_coerce_c_locale(_PyPreConfig *config)
{
if (!config->configure_locale) {
config->coerce_c_locale = 0;
config->coerce_c_locale_warn = 0;
return;
}

const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE");
if (env) {
if (strcmp(env, "0") == 0) {
Expand Down Expand Up @@ -746,7 +755,9 @@ _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args)
}

/* Set LC_CTYPE to the user preferred locale */
_Py_SetLocaleFromEnv(LC_CTYPE);
if (config->configure_locale) {
_Py_SetLocaleFromEnv(LC_CTYPE);
}

_PyPreCmdline cmdline = _PyPreCmdline_INIT;
int init_utf8_mode = Py_UTF8Mode;
Expand Down Expand Up @@ -879,12 +890,14 @@ _PyPreConfig_Write(const _PyPreConfig *config)

_PyPreConfig_SetGlobalConfig(config);

if (config->coerce_c_locale) {
_Py_CoerceLegacyLocale(config->coerce_c_locale_warn);
}
if (config->configure_locale) {
if (config->coerce_c_locale) {
_Py_CoerceLegacyLocale(config->coerce_c_locale_warn);
}

/* Set LC_CTYPE to the user preferred locale */
_Py_SetLocaleFromEnv(LC_CTYPE);
/* Set LC_CTYPE to the user preferred locale */
_Py_SetLocaleFromEnv(LC_CTYPE);
}

/* Write the new pre-configuration into _PyRuntime */
PyMemAllocatorEx old_alloc;
Expand Down