Coverage for mfutil/misc.py: 59%
139 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-12-18 16:04 +0000
1"""Generic utility classes and functions."""
3import uuid
4import errno
5import os
6import logging
7import tempfile
8import socket
9import psutil
10import pickle
11import hashlib
12import datetime
13import time
14import fnmatch
16from inotify_simple import flags
17from mfutil.exc import MFUtilException
18from mfutil.eval import SandboxedEval
20__pdoc__ = {
21 "add_inotify_watch": False
22}
25def __get_logger():
26 return logging.getLogger("mfutil")
29def eval(expr, variables=None):
30 """Evaluate (safely) a python expression (as a string).
32 The eval is done with simpleeval library.
34 Following functions are available (in expressions):
36 - re_match: see match() function of re module
37 - re_imatch: insensitive match() function of re module
38 - fnmatch.fnmatch: fnmatch() function of fnmatch module
40 Args:
41 expr (string): the python expression to eval.
42 variables (dict): if set, inject some variables/values
43 in the expression.
45 """
47 s = SandboxedEval(names=variables)
48 return s.eval(expr)
51def get_unique_hexa_identifier():
52 """Return an unique hexa identifier on 32 bytes.
54 The idenfier is made only with 0123456789abcdef
55 characters.
57 Returns:
58 (string) unique hexa identifier.
60 """
61 return str(uuid.uuid4()).replace('-', '')
64def get_utc_unix_timestamp():
65 """Return the current unix UTC timestamp on all platforms.
67 It works even if the machine is configured in local time.
69 Returns:
70 (int) a int corresponding to the current unix utc timestamp.
72 """
73 dts = datetime.datetime.utcnow()
74 return int(time.mktime(dts.timetuple()))
77def mkdir_p(path, nodebug=False, nowarning=False):
78 """Make a directory recursively (clone of mkdir -p).
80 Thanks to http://stackoverflow.com/questions/600268/
81 mkdir-p-functionality-in-python .
83 Any exceptions are catched and a warning message
84 is logged in case of problems.
86 If the directory already exists, True is returned
87 with no debug or warning.
89 Args:
90 path (string): complete path to create.
91 nodebug (boolean): if True, no debug messages are logged.
92 nowarning (boolean): if True, no message are logged in
93 case of problems.
95 Returns:
96 boolean: True if the directory exists at the end.
98 """
99 try:
100 os.makedirs(path)
101 except OSError as exc:
102 if exc.errno == errno.EEXIST and os.path.isdir(path):
103 return True
104 else:
105 if not nowarning:
106 __get_logger().warning("can't create %s directory", path)
107 return False
108 #We do not log debug message because it is logged in circus configuration file
109 #if not nodebug:
110 # __get_logger().debug("%s directory created", path)
111 return True
114def mkdir_p_or_die(path, nodebug=False, exit_code=2):
115 """Make a directory recursively (clone of mkdir -p).
117 If the directory already exists, True is returned
118 with no debug or warning.
120 Any exceptions are catched.
122 In case of problems, the program dies here with corresponding
123 exit_code.
125 Args:
126 path (string): complete path to create.
127 nodebug (boolean): if True, no debug messages are logged.
128 exit_code (int): os._exit() exit code.
130 """
131 res = mkdir_p(path, nodebug=nodebug, nowarning=True)
132 if not res:
133 __get_logger().error("can't create %s directory", path)
134 os._exit(exit_code)
137def _get_temp_dir(tmp_dir=None):
138 """Return system temp dir or used choosen temp dir.
140 If the user provides a tmp_dir argument, the
141 directory is created (if necessary).
143 If the user don't provide a tmp_dir argument,
144 the function returns a system temp dir.
146 If the directory is not good or can be created,
147 an exception is raised.
149 Args:
150 tmp_dir (string): user provided tmp directory (None
151 to use the system temp dir).
153 Returns:
154 (string) temp directory
156 Raises:
157 MFUtilException if the temp directory is not good or can't
158 be created.
160 """
161 if tmp_dir is None:
162 tmp_dir = tempfile.gettempdir()
163 res = mkdir_p(tmp_dir)
164 if not res:
165 raise MFUtilException("can't create temp_dir: %s", tmp_dir)
166 return tmp_dir
169def get_tmp_filepath(tmp_dir=None, prefix=""):
170 """Return a tmp (complete) filepath.
172 The filename is made with get_unique_hexa_identifier() identifier
173 so 32 hexa characters.
175 The dirname can be provided by the user (or be a system one).
176 He will be created if necessary. An exception can be raised if any
177 problems at this side.
179 Note: the file is not created or open at all. The function just
180 returns a filename.
182 Args:
183 tmp_dir (string): user provided tmp directory (None
184 to use the system temp dir).
185 prefix (string): you can add here a prefix for filenames
186 (will be preprended before the 32 hexa characters).
188 Returns:
189 (string) tmp (complete) filepath.
191 Raises:
192 MFUtilException if the temp directory is not good or can't
193 be created.
195 """
196 temp_dir = _get_temp_dir(tmp_dir)
197 return os.path.join(temp_dir, prefix + get_unique_hexa_identifier())
200def create_tmp_dirpath(tmp_dir=None, prefix=""):
201 """Create and return a temporary directory inside a father tempory directory.
203 The dirname is made with get_unique_hexa_identifier() identifier
204 so 32 hexa characters.
206 The father dirname can be provided by the user (or be a system one).
207 He will be created if necessary. An exception can be raised if any
208 problems at this side.
210 Note: the temporary directory is created.
212 Args:
213 tmp_dir (string): user provided tmp directory (None
214 to use the system temp dir).
215 prefix (string): you can add here a prefix for dirnames
216 (will be preprended before the 32 hexa characters).
218 Returns:
219 (string) complete path of a newly created temporary directory.
221 Raises:
222 MFUtilException if the temp directory can't be created.
224 """
225 temp_dir = _get_temp_dir(tmp_dir)
226 new_temp_dir = os.path.join(temp_dir,
227 prefix + get_unique_hexa_identifier())
228 res = mkdir_p(new_temp_dir, nowarning=True)
229 if not res:
230 raise MFUtilException("can't create temp_dir: %s", new_temp_dir)
231 return new_temp_dir
234def get_ipv4_for_hostname(hostname, static_mappings={}):
235 """Translate a host name to IPv4 address format.
237 The IPv4 address is returned as a string, such as '100.50.200.5'.
238 If the host name is an IPv4 address itself it is returned unchanged.
240 You can provide a dictionnary with static mappings.
241 Following mappings are added by default:
242 '127.0.0.1' => '127.0.0.1'
243 'localhost' => '127.0.0.1'
244 'localhost.localdomain' => '127.0.0.1'
246 Args:
247 hostname (string): hostname.
248 static_mappings (dict): dictionnary of static mappings
249 ((hostname) string: (ip) string).
251 Returns:
252 (string) IPv4 address for the given hostname (None if any problem)
254 """
255 hostname = hostname.lower()
256 static_mappings.update({'127.0.0.1': '127.0.0.1', 'localhost': '127.0.0.1',
257 'localhost.localdomain': '127.0.0.1'})
258 if hostname in static_mappings:
259 return static_mappings[hostname]
260 try:
261 return socket.gethostbyname(hostname)
262 except Exception:
263 return None
266def get_recursive_mtime(directory, ignores=[]):
267 """Get the latest mtime recursivly on a directory.
269 Args:
270 directory (string): complete path of a directory to scan.
271 ignores (list of strings): list of shell-style wildcards
272 to define which filenames/dirnames to ignores (see fnmatch).
274 Returns:
275 (int) timestamp of the latest mtime on the directory.
277 """
278 result = 0
279 for name in os.listdir(directory):
280 ignored = False
281 for ssw in ignores:
282 if fnmatch.fnmatch(name, ssw):
283 ignored = True
284 break
285 if ignored:
286 continue
287 fullpath = os.path.join(directory, name)
288 if os.path.isdir(fullpath):
289 mtime = get_recursive_mtime(fullpath, ignores=ignores)
290 else:
291 mtime = 0
292 try:
293 mtime = int(os.path.getmtime(fullpath))
294 except Exception:
295 pass
296 if mtime > result:
297 result = mtime
298 return result
301def add_inotify_watch(inotify, directory, ignores=[]):
302 """Register recursively directories to watch.
304 Args:
305 inotify (inotify object): object that owns the file descriptors
306 directory (string): complete path of a directory to scan.
307 ignores (list of strings): list of shell-style wildcards
308 to define which filenames/dirnames to ignores (see fnmatch).
310 """
311 watch_flags = flags.MODIFY | flags.CREATE |\
312 flags.DELETE | flags.DELETE_SELF
313 try:
314 __get_logger().info("watch %s" % directory)
315 inotify.add_watch(directory, watch_flags)
316 except Exception as e:
317 __get_logger().warning("cannot watch %s: %s" % (directory, e))
319 if not os.access(directory, os.R_OK):
320 __get_logger().warning("cannot enter into %s" % directory)
321 return
323 for name in os.listdir(directory):
324 ignored = False
325 for ssw in ignores:
326 if fnmatch.fnmatch(name, ssw):
327 ignored = True
328 break
329 if ignored:
330 continue
331 fullpath = os.path.join(directory, name)
332 if os.path.isdir(fullpath):
333 add_inotify_watch(inotify, fullpath, ignores=ignores)
336def _kill_process_and_children(process):
337 children = None
338 all_children = None
339 # First we keep the full view of the process tree to kill
340 try:
341 all_children = process.children(recursive=True)
342 except psutil.NoSuchProcess:
343 pass
344 # Then, we keep just immediate children to kill in the "good" order
345 try:
346 children = process.children(recursive=False)
347 except psutil.NoSuchProcess:
348 pass
349 try:
350 process.kill()
351 except psutil.NoSuchProcess:
352 pass
353 if children is not None:
354 for child in children:
355 _kill_process_and_children(child)
356 # To be sure, we didn't miss something, we kill the initial full list
357 if all_children is not None:
358 for child in all_children:
359 _kill_process_and_children(child)
362def kill_process_and_children(pid):
363 """Kill recursively a complete tree of processes.
365 Given a pid, this method recursively kills the complete tree (children and
366 children of each child...) of this process.
368 The SIGKILL signal is used.
370 Args:
371 pid (int): process PID to kill.
373 """
374 try:
375 process = psutil.Process(pid)
376 except psutil.NoSuchProcess:
377 return
378 _kill_process_and_children(process)
381def hash_generator(*args):
382 """Generate a hash from a variable number of arguments as a safe string.
384 Note that pickle is used so arguments have to be serializable.
386 Args:
387 *args: arguments to hash
389 """
390 temp = pickle.dumps(args, pickle.HIGHEST_PROTOCOL)
391 return hashlib.md5(temp).hexdigest()