Coverage for mflog/utils.py: 65%

1# -*- coding: utf-8 -*-

3from __future__ import print_function

4import os

5import fnmatch

6import logging

7import fcntl

8import sys

9import six

10import importlib

11import inspect

12try:

13 from rich.console import Console

14 from rich.tabulate import tabulate_mapping

15except ImportError:

16 pass

18OVERRIDE_LINES_CACHE = None

19LEVEL_FROM_LOGGER_NAME_CACHE = {}

22def write_with_lock(f, message):

23 fcntl.flock(f, fcntl.LOCK_EX)

24 f.write(message)

27def flush_with_lock(f):

28 f.flush()

29 fcntl.flock(f, fcntl.LOCK_UN)

32def __reset_level_from_logger_name_cache():

33 global LEVEL_FROM_LOGGER_NAME_CACHE

34 LEVEL_FROM_LOGGER_NAME_CACHE = {}

37def get_func_by_path(func_path):

38 func_name = func_path.split('.')[-1]

39 module_path = ".".join(func_path.split('.')[0:-1])

40 if module_path == "": 40 ↛ 41line 40 didn't jump to line 41, because the condition on line 40 was never true

41 print("ERROR: %s must follow 'pkg.function_name'" % func_path,

42 file=sys.stderr)

43 sys.exit(1)

44 if func_path.endswith(')'): 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 print("ERROR: %s must follow 'pkg.function_name'" % func_path,

46 file=sys.stderr)

47 print("=> EXIT", file=sys.stderr)

48 sys.exit(1)

49 try:

50 mod = importlib.import_module(module_path)

51 except Exception:

52 print("ERROR: can't import %s" % module_path, file=sys.stderr)

53 print("=> EXIT", file=sys.stderr)

54 sys.exit(1)

55 try:

56 return getattr(mod, func_name)

57 except Exception:

58 print("ERROR: can't get %s on %s" % (func_name, module_path),

59 file=sys.stderr)

60 print("=> EXIT", file=sys.stderr)

61 sys.exit(1)

64class classproperty(object):

66 def __init__(self, f):

67 self.f = f

69 def __get__(self, obj, owner):

70 return self.f(owner)

73class Config(object):

75 _instance = None

76 _minimal_level = None

77 _json_minimal_level = None

78 _json_file = None

79 _override_files = None

80 _override_dict = None

81 _extra_context_func = None

82 _json_only_keys = None

83 _syslog_address = None

84 _syslog_format = None

85 _syslog_minimal_level = None

86 _fancy_output = None

87 _auto_dump_locals = True

89 def __init__(self, minimal_level=None, json_minimal_level=None,

90 json_file=None, override_files=None,

91 thread_local_context=False,

92 extra_context_func=None, json_only_keys=None,

93 override_dict={}, syslog_address=None, syslog_format=None,

94 syslog_minimal_level=None,

95 fancy_output=None,

96 auto_dump_locals=True):

97 global LEVEL_FROM_LOGGER_NAME_CACHE, OVERRIDE_LINES_CACHE

98 OVERRIDE_LINES_CACHE = {}

99 LEVEL_FROM_LOGGER_NAME_CACHE = {}

100 if minimal_level is not None: 100 ↛ 101line 100 didn't jump to line 101, because the condition on line 100 was never true

101 self._minimal_level = minimal_level

102 else:

103 self._minimal_level = os.environ.get('MFLOG_MINIMAL_LEVEL', 'INFO')

104 if json_minimal_level is not None: 104 ↛ 105line 104 didn't jump to line 105, because the condition on line 104 was never true

105 self._json_minimal_level = json_minimal_level

106 else:

107 self._json_minimal_level = \

108 os.environ.get('MFLOG_JSON_MINIMAL_LEVEL', 'WARNING')

109 if syslog_minimal_level is not None: 109 ↛ 110line 109 didn't jump to line 110, because the condition on line 109 was never true

110 self._syslog_minimal_level = syslog_minimal_level

111 else:

112 self._syslog_minimal_level = \

113 os.environ.get('MFLOG_SYSLOG_MINIMAL_LEVEL', 'WARNING')

114 if syslog_format is not None: 114 ↛ 115line 114 didn't jump to line 115, because the condition on line 114 was never true

115 self._syslog_format = syslog_format

116 else:

117 self._syslog_format = \

118 os.environ.get('MFLOG_SYSLOG_FORMAT', 'null')

119 if self._syslog_format not in ('null', 'msg_only', 'json'): 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 raise Exception("unknown syslog format: %s => must be null, "

121 "msg_only or json")

122 if self._syslog_format == "null": 122 ↛ 124line 122 didn't jump to line 124, because the condition on line 122 was never false

123 self._syslog_format = None

124 if json_file is not None: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 self._json_file = json_file

126 else:

127 self._json_file = os.environ.get("MFLOG_JSON_FILE", None)

128 if self._json_file == "null": 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true

129 self._json_file = None

130 if syslog_address is not None: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true

131 tmpsyslog = syslog_address

132 else:

133 tmpsyslog = os.environ.get("MFLOG_SYSLOG_ADDRESS", None)

134 if tmpsyslog == "null": 134 ↛ 135line 134 didn't jump to line 135, because the condition on line 134 was never true

135 tmpsyslog = None

136 if isinstance(tmpsyslog, six.string_types): 136 ↛ 137line 136 didn't jump to line 137, because the condition on line 136 was never true

137 tmpsyslog2 = tmpsyslog.split(':')

138 if len(tmpsyslog2) == 1:

139 self._syslog_address = (tmpsyslog2[0], 514)

140 elif len(tmpsyslog2) == 2:

141 self._syslog_address = (tmpsyslog2[0], int(tmpsyslog2[1]))

142 else:

143 raise Exception("wrong syslog_address type: %s" % tmpsyslog)

144 if override_files is not None: 144 ↛ 145line 144 didn't jump to line 145, because the condition on line 144 was never true

145 self._override_files = override_files

146 else:

147 self._override_files = \

148 [x.strip() for x in os.environ.get(

149 "MFLOG_MINIMAL_LEVEL_OVERRIDE_FILES", "").split(';')]

150 if extra_context_func is not None:

151 self._extra_context_func = extra_context_func

152 else:

153 if "MFLOG_EXTRA_CONTEXT_FUNC" in os.environ:

154 self._extra_context_func = \

155 get_func_by_path(os.environ['MFLOG_EXTRA_CONTEXT_FUNC'])

156 else:

157 self._extra_context_func = None

158 if self._extra_context_func and not callable(self._extra_context_func): 158 ↛ 159line 158 didn't jump to line 159, because the condition on line 158 was never true

159 print("ERROR: extra_context_func must be a python callable",

160 file=sys.stderr)

161 print("=> EXIT", file=sys.stderr)

162 sys.exit(1)

163 if json_only_keys is not None:

164 self._json_only_keys = json_only_keys

165 else:

166 if "MFLOG_JSON_ONLY_KEYS" in os.environ:

167 self._json_only_keys = \

168 os.environ["MFLOG_JSON_ONLY_KEYS"].split(',')

169 else:

170 self._json_only_keys = []

171 self._override_dict = override_dict

172 if fancy_output is None: 172 ↛ 178line 172 didn't jump to line 178, because the condition on line 172 was never false

173 if 'rich' in sys.modules: 173 ↛ 176line 173 didn't jump to line 176, because the condition on line 173 was never false

174 self._fancy_output = None

175 else:

176 self._fancy_output = False

177 else:

178 self._fancy_output = fancy_output

179 self._auto_dump_locals = auto_dump_locals

180

181 @classmethod

182 def get_instance(cls):

183 if cls._instance is None: 183 ↛ 184line 183 didn't jump to line 184, because the condition on line 183 was never true

184 cls._instance = Config()

185 return cls._instance

186

187 @classmethod

188 def set_instance(cls, *args, **kwargs):

189 cls._instance = Config(*args, **kwargs)

190

191 @classproperty

192 def minimal_level(cls): # pylint: disable=E0213

193 return cls.get_instance()._minimal_level

194

195 @classproperty

196 def auto_dump_locals(cls): # pylint: disable=E0213

197 return cls.get_instance()._auto_dump_locals

198

199 @classproperty

200 def extra_context_func(cls): # pylint: disable=E0213

201 return cls.get_instance()._extra_context_func

202

203 @classproperty

204 def json_minimal_level(cls): # pylint: disable=E0213

205 return cls.get_instance()._json_minimal_level

206

207 @classproperty

208 def json_file(cls): # pylint: disable=E0213

209 return cls.get_instance()._json_file

210

211 @classproperty

212 def override_files(cls): # pylint: disable=E0213

213 return cls.get_instance()._override_files

214

215 @classproperty

216 def syslog_minimal_level(cls): # pylint: disable=E0213

217 return cls.get_instance()._syslog_minimal_level

218

219 @classproperty

220 def syslog_address(cls): # pylint: disable=E0213

221 return cls.get_instance()._syslog_address

222

223 @classproperty

224 def syslog_format(cls): # pylint: disable=E0213

225 return cls.get_instance()._syslog_format

226

227 @classproperty

228 def override_dict(cls): # pylint: disable=E0213

229 return cls.get_instance()._override_dict

230

231 @classproperty

232 def json_only_keys(cls): # pylint: disable=E0213

233 return cls.get_instance()._json_only_keys

234

235 @classproperty

236 def fancy_output(cls): # pylint: disable=E0213

237 return cls.get_instance()._fancy_output

238

239

240def level_name_to_level_no(level_name):

241 """Convert level_name (debug, WARNING...) to level number.

242

243 Args:

244 level_name (string): A level name (debug, warning, info, critical

245 or errot), case insensitive.

246

247 Returns:

248 (int) Corresponding level number (in logging library).

249

250 Raises:

251 Exception: if the level in unknown.

252

253 """

254 ulevel_name = level_name.upper()

255 if ulevel_name == "DEBUG" or ulevel_name == "NOTSET":

256 return logging.DEBUG

257 elif ulevel_name == "INFO":

258 return logging.INFO

259 elif ulevel_name == "WARNING":

260 return logging.WARNING

261 elif ulevel_name == "ERROR" or ulevel_name == "EXCEPTION":

262 return logging.ERROR

263 elif ulevel_name == "CRITICAL": 263 ↛ 266line 263 didn't jump to line 266, because the condition on line 263 was never false

264 return logging.CRITICAL

265 else:

266 raise Exception("unknown level name: %s" % level_name)

267

268

269def _file_to_lines(file_path):

270 """Read the given file_path and decode mflog_override format.

271

272 foo.bar.* => DEBUG

273 foo.* => WARNING

274

275 Notes:

276 - lines beginning with # are ignored

277 - the left part is a read as a fnmatch pattern

278 - the right part is a case insensitive level name

279

280 Args:

281 file_path (string): The full path of the file to read.

282

283 Returns:

284 (list of couples) A list of (logger name pattern, level name).

285

286 """

287 try:

288 with open(file_path, "r") as f:

289 tmp = [x.split('=>') for x in f.readlines()]

290 lines = [(x[0].strip(), x[1].strip()) for x in tmp

291 if len(x) == 2 and not x[0].strip().startswith('#')]

292 return lines

293 except IOError:

294 return []

295

296

297def _get_override_lines(path):

298 """Read the custom level override configuration file (if exists).

299

300 Args:

301 path (string): the full path of the override configuration file.

302

303 Note: the content is cached (in memory).

304

305 Returns:

306 (list of couples) A list of (logger name pattern, level name).

307

308 """

309 global OVERRIDE_LINES_CACHE

310 if path not in OVERRIDE_LINES_CACHE: 310 ↛ 313line 310 didn't jump to line 313, because the condition on line 310 was never false

311 lines = _file_to_lines(path)

312 OVERRIDE_LINES_CACHE[path] = lines

313 return OVERRIDE_LINES_CACHE[path]

314

315

316def get_extra_context():

317 """Return an extra context by calling an external configured python func.

318

319 Returns:

320 (dict) A dict of extra context key/values as strings.

321

322 """

323 extra_context_f = Config.extra_context_func

324 if extra_context_f is None:

325 return {}

326 extra_context = extra_context_f() # pylint: disable=E1120

327 if not isinstance(extra_context, dict): 327 ↛ 328line 327 didn't jump to line 328, because the condition on line 327 was never true

328 print("bad extra_context (not a dict) => ignoring", file=sys.stderr)

329 return {}

330 return extra_context

331

332

333def get_level_no_from_logger_name(logger_name):

334 """Get the level number to use for the given logger name.

335

336 Note:

337 - first we search in override_dict, if there is a match, it's over

338 - (if no match) at first step, we check each files in override_files

339 configuration. The first match wins.

340 - if there is no match, we return the default level number.

341

342 Note: the result is cached in memory.

343

344 Args:

345 logger_name (string): The logger name.

346

347 Returns:

348 (int) The level number to use for this logger name.

349

350 """

351 global LEVEL_FROM_LOGGER_NAME_CACHE

352

353 class Found(Exception):

354 pass

355

356 if logger_name not in LEVEL_FROM_LOGGER_NAME_CACHE:

357 # pylint: disable=no-member

358 for k, v in Config.override_dict.items():

359 if fnmatch.fnmatch(logger_name, k):

360 LEVEL_FROM_LOGGER_NAME_CACHE[logger_name] = \

361 level_name_to_level_no(v)

362 return LEVEL_FROM_LOGGER_NAME_CACHE[logger_name]

363 paths = Config.override_files

364 try:

365 for path in paths: # pylint: disable=E1133

366 custom_lines = _get_override_lines(path)

367 for k, v in custom_lines: 367 ↛ 368line 367 didn't jump to line 368, because the loop on line 367 never started

368 if fnmatch.fnmatch(logger_name, k):

369 LEVEL_FROM_LOGGER_NAME_CACHE[logger_name] = \

370 level_name_to_level_no(v)

371 raise Found

372 except Found:

373 pass

374 else:

375 LEVEL_FROM_LOGGER_NAME_CACHE[logger_name] = \

376 level_name_to_level_no(Config.minimal_level)

377 return LEVEL_FROM_LOGGER_NAME_CACHE[logger_name]

378

379

380def get_resolved_fancy_output_config_value(f=sys.stderr):

381 fancy = Config.fancy_output

382 if fancy is None: 382 ↛ 387line 382 didn't jump to line 387, because the condition on line 382 was never false

383 try:

384 fancy = f.isatty()

385 except Exception:

386 fancy = False

387 return fancy

388

389

390def dump_locals(f=sys.stderr):

391 fancy = get_resolved_fancy_output_config_value(f=f)

392 stack_offset = -1

393 try:

394 caller = inspect.stack()[stack_offset]

395 locals_map = {

396 key: value

397 for key, value in caller.frame.f_locals.items()

398 if not key.startswith("__")

399 }

400 for k, v in locals_map.items():

401 if len(repr(v)) > 10000:

402 locals_map[k] = \

403 "(too big value => hidden in this variables dump)"

404 if fancy:

405 c = Console(file=sys.stderr)

406 c.print(tabulate_mapping(locals_map, title="Locals"))

407 else:

408 print("Locals dump", file=f)

409 for k, v in locals_map.items():

410 print("%s: %r" % (k, v))

411 except Exception:

412 return False

413 return True