userver: /data/code/userver/testsuite/pytest_plugins/pytest_userver/plugins/config.py Source File
Loading...
Searching...
No Matches
config.py
1"""
2Work with the configuration files of the service in testsuite.
3"""
4
5# pylint: disable=redefined-outer-name
6import copy
7import logging
8import os
9import pathlib
10import types
11import typing
12
13import pytest
14import yaml
15
16
17# flake8: noqa E266
18
32USERVER_CONFIG_HOOKS = [
33 'userver_config_http_server',
34 'userver_config_http_client',
35 'userver_config_logging',
36 'userver_config_testsuite',
37 'userver_config_secdist',
38 'userver_config_testsuite_middleware',
39]
40
41
42# @cond
43
44
45logger = logging.getLogger(__name__)
46
47
48class _UserverConfigPlugin:
49 def __init__(self):
50 self._config_hooks = []
51
52 @property
53 def userver_config_hooks(self):
54 return self._config_hooks
55
56 def pytest_plugin_registered(self, plugin, manager):
57 if not isinstance(plugin, types.ModuleType):
58 return
59 uhooks = getattr(plugin, 'USERVER_CONFIG_HOOKS', None)
60 if uhooks is not None:
61 self._config_hooks.extend(uhooks)
62
63
64class _UserverConfig(typing.NamedTuple):
65 config_yaml: dict
66 config_vars: dict
67
68
69def pytest_configure(config):
70 config.pluginmanager.register(_UserverConfigPlugin(), 'userver_config')
71 config.addinivalue_line(
72 'markers', 'config: per-test dynamic config values',
73 )
74
75
76def pytest_addoption(parser) -> None:
77 group = parser.getgroup('userver-config')
78 group.addoption(
79 '--service-log-level',
80 type=str.lower,
81 choices=['trace', 'debug', 'info', 'warning', 'error', 'critical'],
82 )
83 group.addoption(
84 '--service-config',
85 type=pathlib.Path,
86 help='Path to service.yaml file.',
87 )
88 group.addoption(
89 '--service-config-vars',
90 type=pathlib.Path,
91 help='Path to config_vars.yaml file.',
92 )
93 group.addoption(
94 '--service-secdist',
95 type=pathlib.Path,
96 help='Path to secure_data.json file.',
97 )
98 group.addoption(
99 '--config-fallback',
100 type=pathlib.Path,
101 help='Path to dynamic config fallback file.',
102 )
103
104
105# @endcond
106
107
108@pytest.fixture(scope='session')
109def service_config_path(pytestconfig) -> pathlib.Path:
110 """
111 Returns the path to service.yaml file set by command line
112 `--service-config` option.
113
114 Override this fixture to change the way path to service.yaml is provided.
115
116 @ingroup userver_testsuite_fixtures
117 """
118 return pytestconfig.option.service_config
119
120
121@pytest.fixture(scope='session')
122def service_config_vars_path(pytestconfig) -> typing.Optional[pathlib.Path]:
123 """
124 Returns the path to config_vars.yaml file set by command line
125 `--service-config-vars` option.
126
127 Override this fixture to change the way path to config_vars.yaml is
128 provided.
129
130 @ingroup userver_testsuite_fixtures
131 """
132 return pytestconfig.option.service_config_vars
133
134
135@pytest.fixture(scope='session')
136def service_secdist_path(pytestconfig) -> typing.Optional[pathlib.Path]:
137 """
138 Returns the path to secure_data.json file set by command line
139 `--service-secdist` option.
140
141 Override this fixture to change the way path to secure_data.json is
142 provided.
143
144 @ingroup userver_testsuite_fixtures
145 """
146 return pytestconfig.option.service_secdist
147
148
149@pytest.fixture(scope='session')
150def config_fallback_path(pytestconfig) -> pathlib.Path:
151 """
152 Returns the path to dynamic config fallback file set by command line
153 `--config-fallback` option.
154
155 Override this fixture to change the way path to dynamic config fallback is
156 provided.
157
158 @ingroup userver_testsuite_fixtures
159 """
160 return pytestconfig.option.config_fallback
161
162
163@pytest.fixture(scope='session')
164def service_tmpdir(service_binary, tmp_path_factory):
165 """
166 Returns the path for temporary files. The path is the same for the whole
167 session and files are not removed (at least by this fixture) between
168 tests.
169
170 @ingroup userver_testsuite_fixtures
171 """
172 return tmp_path_factory.mktemp(pathlib.Path(service_binary).name)
173
174
175@pytest.fixture(scope='session')
177 service_tmpdir, service_config, service_config_yaml,
178) -> pathlib.Path:
179 """
180 Dumps the contents of the service_config_yaml into a static config for
181 testsuite and returns the path to the config file.
182
183 @ingroup userver_testsuite_fixtures
184 """
185 dst_path = service_tmpdir / 'config.yaml'
186
187 logger.debug(
188 'userver fixture "service_config_path_temp" writes the patched static '
189 'config to "%s" equivalent to:\n%s',
190 dst_path,
191 yaml.dump(service_config),
192 )
193 dst_path.write_text(yaml.dump(service_config_yaml))
194
195 return dst_path
196
197
198@pytest.fixture(scope='session')
199def service_config_yaml(_service_config_hooked) -> dict:
200 """
201 Returns the static config values after the USERVER_CONFIG_HOOKS were
202 applied (if any). Prefer using
203 pytest_userver.plugins.config.service_config
204
205 @ingroup userver_testsuite_fixtures
206 """
207 return _service_config_hooked.config_yaml
208
209
210@pytest.fixture(scope='session')
211def service_config_vars(_service_config_hooked) -> dict:
212 """
213 Returns the static config variables (config_vars.yaml) values after the
214 USERVER_CONFIG_HOOKS were applied (if any). Prefer using
215 pytest_userver.plugins.config.service_config
216
217 @ingroup userver_testsuite_fixtures
218 """
219 return _service_config_hooked.config_vars
220
221
222def _substitute_values(config, service_config_vars: dict, service_env) -> None:
223 if isinstance(config, dict):
224 for key, value in config.items():
225 if not isinstance(value, str):
226 _substitute_values(value, service_config_vars, service_env)
227 continue
228
229 if not value.startswith('$'):
230 continue
231
232 new_value = service_config_vars.get(value[1:])
233 if new_value is not None:
234 config[key] = new_value
235 continue
236
237 env = config.get(f'{key}#env')
238 if env:
239 if service_env:
240 new_value = service_env.get(service_env)
241 if not new_value:
242 new_value = os.environ.get(env)
243 if new_value:
244 config[key] = new_value
245 continue
246
247 fallback = config.get(f'{key}#fallback')
248 if fallback:
249 config[key] = fallback
250
251 if isinstance(config, list):
252 for i, value in enumerate(config):
253 if not isinstance(value, str):
254 _substitute_values(value, service_config_vars, service_env)
255 continue
256
257 if not value.startswith('$'):
258 continue
259
260 new_value = service_config_vars.get(value[1:])
261 if new_value is not None:
262 config[i] = new_value
263
264
265@pytest.fixture(scope='session')
267 service_config_yaml, service_config_vars, service_env,
268) -> dict:
269 """
270 Returns the static config values after the USERVER_CONFIG_HOOKS were
271 applied (if any) and with all the '$', environment and fallback variables
272 substituted.
273
274 @ingroup userver_testsuite_fixtures
275 """
276 config = copy.deepcopy(service_config_yaml)
277 _substitute_values(config, service_config_vars, service_env)
278 config.pop('config_vars', None)
279 return config
280
281
282@pytest.fixture(scope='session')
283def _original_service_config(
284 service_config_path, service_config_vars_path,
285) -> _UserverConfig:
286 config_vars: dict
287 config_yaml: dict
288
289 with open(service_config_path, mode='rt') as fp:
290 config_yaml = yaml.safe_load(fp)
291
292 if service_config_vars_path:
293 with open(service_config_vars_path, mode='rt') as fp:
294 config_vars = yaml.safe_load(fp)
295 else:
296 config_vars = {}
297
298 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
299
300
301@pytest.fixture(scope='session')
302def _service_config_hooked(
303 pytestconfig, request, service_tmpdir, _original_service_config,
304) -> _UserverConfig:
305 config_yaml = copy.deepcopy(_original_service_config.config_yaml)
306 config_vars = copy.deepcopy(_original_service_config.config_vars)
307
308 plugin = pytestconfig.pluginmanager.get_plugin('userver_config')
309 for hook in plugin.userver_config_hooks:
310 if not callable(hook):
311 hook_func = request.getfixturevalue(hook)
312 else:
313 hook_func = hook
314 hook_func(config_yaml, config_vars)
315
316 if not config_vars:
317 config_yaml.pop('config_vars', None)
318 else:
319 config_vars_path = service_tmpdir / 'config_vars.yaml'
320 config_vars_text = yaml.dump(config_vars)
321 logger.debug(
322 'userver fixture "service_config" writes the patched static '
323 'config vars to "%s":\n%s',
324 config_vars_path,
325 config_vars_text,
326 )
327 config_vars_path.write_text(config_vars_text)
328 config_yaml['config_vars'] = str(config_vars_path)
329
330 return _UserverConfig(config_yaml=config_yaml, config_vars=config_vars)
331
332
333@pytest.fixture(scope='session')
334def userver_config_http_server(service_port, monitor_port):
335 """
336 Returns a function that adjusts the static configuration file for testsuite.
337 Sets the `server.listener.port` to listen on
338 @ref pytest_userver.plugins.base.service_port "service_port" fixture value;
339 sets the `server.listener-monitor.port` to listen on
340 @ref pytest_userver.plugins.base.monitor_port "monitor_port"
341 fixture value.
342
343 @ingroup userver_testsuite_fixtures
344 """
345
346 def _patch_config(config_yaml, config_vars):
347 components = config_yaml['components_manager']['components']
348 if 'server' in components:
349 server = components['server']
350 if 'listener' in server:
351 server['listener']['port'] = service_port
352
353 if 'listener-monitor' in server:
354 server['listener-monitor']['port'] = monitor_port
355
356 return _patch_config
357
358
359@pytest.fixture(scope='session')
360def allowed_url_prefixes_extra() -> typing.List[str]:
361 """
362 By default, userver HTTP client is only allowed to talk to mockserver
363 when running in testsuite. This makes tests repeatable and encapsulated.
364
365 Override this fixture to whitelist some additional URLs.
366 It is still strongly advised to only talk to localhost in tests.
367
368 @ingroup userver_testsuite_fixtures
369 """
370 return []
371
372
373@pytest.fixture(scope='session')
375 mockserver_info, mockserver_ssl_info, allowed_url_prefixes_extra,
376):
377 """
378 Returns a function that adjusts the static configuration file for testsuite.
379 Sets increased timeout and limits allowed URLs for `http-client` component.
380
381 @ingroup userver_testsuite_fixtures
382 """
383
384 def patch_config(config, config_vars):
385 components: dict = config['components_manager']['components']
386 if not {'http-client', 'testsuite-support'}.issubset(
387 components.keys(),
388 ):
389 return
390 http_client = components['http-client'] or {}
391 http_client['testsuite-enabled'] = True
392 http_client['testsuite-timeout'] = '10s'
393
394 allowed_urls = [mockserver_info.base_url]
395 if mockserver_ssl_info:
396 allowed_urls.append(mockserver_ssl_info.base_url)
397 allowed_urls += allowed_url_prefixes_extra
398 http_client['testsuite-allowed-url-prefixes'] = allowed_urls
399
400 return patch_config
401
402
403@pytest.fixture(scope='session')
405 """
406 Default log level to use in userver if no caoomand line option was provided.
407
408 Returns 'debug'.
409
410 @ingroup userver_testsuite_fixtures
411 """
412 return 'debug'
413
414
415@pytest.fixture(scope='session')
416def userver_log_level(pytestconfig, userver_default_log_level) -> str:
417 """
418 Returns --service-log-level value if provided, otherwise returns
419 userver_default_log_level() value from fixture.
420
421 @ingroup userver_testsuite_fixtures
422 """
423 if pytestconfig.option.service_log_level:
424 return pytestconfig.option.service_log_level
425 return userver_default_log_level
426
427
428@pytest.fixture(scope='session')
429def userver_config_logging(userver_log_level, _service_logfile_path):
430 """
431 Returns a function that adjusts the static configuration file for testsuite.
432 Sets the `logging.loggers.default` to log to `@stderr` with level set
433 from `--service-log-level` pytest configuration option.
434
435 @ingroup userver_testsuite_fixtures
436 """
437
438 if _service_logfile_path:
439 default_file_path = str(_service_logfile_path)
440 else:
441 default_file_path = '@stderr'
442
443 def _patch_config(config_yaml, config_vars):
444 components = config_yaml['components_manager']['components']
445 if 'logging' in components:
446 loggers = components['logging'].setdefault('loggers', {})
447 for logger in loggers.values():
448 logger['file_path'] = '@null'
449 loggers['default'] = {
450 'file_path': default_file_path,
451 'level': userver_log_level,
452 'overflow_behavior': 'discard',
453 }
454 config_vars['logger_level'] = userver_log_level
455
456 return _patch_config
457
458
459@pytest.fixture(scope='session')
460def userver_config_testsuite(pytestconfig, mockserver_info):
461 """
462 Returns a function that adjusts the static configuration file for testsuite.
463
464 Sets up `testsuite-support` component, which:
465
466 - increases timeouts for userver drivers
467 - disables periodic cache updates
468 - enables testsuite tasks
469
470 Sets the `testsuite-enabled` in config_vars.yaml to `True`; sets the
471 `tests-control.testpoint-url` to mockserver URL.
472
473 @ingroup userver_testsuite_fixtures
474 """
475
476 def _set_postgresql_options(testsuite_support: dict) -> None:
477 testsuite_support['testsuite-pg-execute-timeout'] = '35s'
478 testsuite_support['testsuite-pg-statement-timeout'] = '30s'
479 testsuite_support['testsuite-pg-readonly-master-expected'] = True
480
481 def _set_redis_timeout(testsuite_support: dict) -> None:
482 testsuite_support['testsuite-redis-timeout-connect'] = '40s'
483 testsuite_support['testsuite-redis-timeout-single'] = '30s'
484 testsuite_support['testsuite-redis-timeout-all'] = '30s'
485
486 def _disable_cache_periodic_update(testsuite_support: dict) -> None:
487 testsuite_support['testsuite-periodic-update-enabled'] = False
488
489 def patch_config(config, config_vars) -> None:
490 components: dict = config['components_manager']['components']
491 if 'testsuite-support' not in components:
492 return
493 testsuite_support = components['testsuite-support'] or {}
494 testsuite_support['testsuite-increased-timeout'] = '30s'
495 _set_postgresql_options(testsuite_support)
496 _set_redis_timeout(testsuite_support)
497 service_runner = pytestconfig.getoption('--service-runner-mode', False)
498 if not service_runner:
499 _disable_cache_periodic_update(testsuite_support)
500 testsuite_support['testsuite-tasks-enabled'] = not service_runner
501 testsuite_support[
502 'testsuite-periodic-dumps-enabled'
503 ] = '$userver-dumps-periodic'
504 components['testsuite-support'] = testsuite_support
505
506 config_vars['testsuite-enabled'] = True
507 if 'tests-control' in components:
508 components['tests-control']['testpoint-url'] = mockserver_info.url(
509 'testpoint',
510 )
511
512 return patch_config
513
514
515@pytest.fixture(scope='session')
516def userver_config_secdist(service_secdist_path):
517 """
518 Returns a function that adjusts the static configuration file for testsuite.
519 Sets the `default-secdist-provider.config` to the value of
520 @ref pytest_userver.plugins.config.service_secdist_path "service_secdist_path"
521 fixture.
522
523 @ingroup userver_testsuite_fixtures
524 """
525
526 def _patch_config(config_yaml, config_vars):
527 if not service_secdist_path:
528 return
529
530 components = config_yaml['components_manager']['components']
531 if 'default-secdist-provider' not in components:
532 return
533
534 if not service_secdist_path.is_file():
535 raise ValueError(
536 f'"{service_secdist_path}" is not a file. Provide a '
537 f'"--service-secdist" pytest option or override the '
538 f'"service_secdist_path" fixture.',
539 )
540 components['default-secdist-provider']['config'] = str(
541 service_secdist_path,
542 )
543
544 return _patch_config
545
546
547@pytest.fixture(scope='session')
548def userver_config_testsuite_middleware(
549 userver_testsuite_middleware_enabled: bool,
550):
551 def patch_config(config_yaml, config_vars):
552 if not userver_testsuite_middleware_enabled:
553 return
554
555 components = config_yaml['components_manager']['components']
556 if 'server' not in components:
557 return
558
559 pipeline_builder = components.setdefault(
560 'default-server-middleware-pipeline-builder', {},
561 )
562 middlewares = pipeline_builder.setdefault('append', [])
563 middlewares.append('testsuite-exceptions-handling-middleware')
564
565 return patch_config
566
567
568@pytest.fixture(scope='session')
570 """Enabled testsuite middleware."""
571 return True