Coverage for mfplugin/plugin.py: 79%
319 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-11-13 15:57 +0000
1import os
2import hashlib
3import json
4from datetime import datetime, timezone
5import pickle
6from pathlib import Path
7import inspect
8import shutil
9import socket
10from gitignore_parser import parse_gitignore
11from mfutil import BashWrapper, get_unique_hexa_identifier, mkdir_p_or_die, \
12 mkdir_p, BashWrapperOrRaise, hash_generator
13from mfplugin.configuration import Configuration
14from mfplugin.app import App
15from mfplugin.extra_daemon import ExtraDaemon
16from mfplugin.utils import BadPlugin, get_default_plugins_base_dir, \
17 layerapi2_label_file_to_plugin_name, validate_plugin_name, \
18 CantBuildPlugin, get_current_envs, PluginEnvContextManager, \
19 get_configuration_class, get_app_class, get_extra_daemon_class, \
20 get_configuration_paths, \
21 is_jsonable, layerapi2_label_to_plugin_home, plugin_name_to_layerapi2_label
23MFEXT_HOME = os.environ.get("MFEXT_HOME", None)
24MFMODULE_RUNTIME_HOME = os.environ.get('MFMODULE_RUNTIME_HOME', '/tmp')
25MFMODULE_LOWERCASE = os.environ.get('MFMODULE_LOWERCASE', 'generic')
26MFMODULE = os.environ.get('MFMODULE', 'GENERIC')
27SPEC_TEMPLATE = os.path.join(os.path.dirname(os.path.realpath(__file__)),
28 "plugin.spec")
29BUID_HOST = os.environ.get('MFHOSTNAME_FULL', socket.gethostname())
32class Plugin(object):
34 def __init__(self, plugins_base_dir, home,
35 configuration_class=None,
36 extra_daemon_class=None,
37 app_class=None,
38 dont_read_config_overrides=False):
39 self.configuration_class = get_configuration_class(configuration_class,
40 Configuration)
41 """Configuration class."""
42 self.app_class = get_app_class(app_class, App)
43 """App class."""
44 self.extra_daemon_class = get_extra_daemon_class(extra_daemon_class,
45 ExtraDaemon)
46 """Extra Daemon class."""
47 self.home = os.path.abspath(home)
48 """Plugin home (absolute and normalized string)."""
49 self.plugins_base_dir = plugins_base_dir \
50 if plugins_base_dir is not None else get_default_plugins_base_dir()
51 """Plugin base directory (string)."""
52 self.name = self._get_name()
53 """Plugin name (string)."""
54 self.is_dev_linked = os.path.islink(os.path.join(self.plugins_base_dir,
55 self.name))
56 """Is the plugin a devlink? (boolean)."""
57 self._dont_read_config_overrides = dont_read_config_overrides
58 self._metadata = {}
59 self._files = None
60 self.__loaded = False
61 # FIXME: detect broken symlink
63 def _get_debug(self):
64 self.load()
65 self._load_files() # as it is not included in load() for perfs reasons
66 res = {x: y for x, y in inspect.getmembers(self)
67 if is_jsonable(y) and not x.startswith('_')}
68 res['configuration'] = self.configuration._get_debug()
69 return res
71 def _get_name(self):
72 llfpath = os.path.join(self.home, ".layerapi2_label")
73 tmp = layerapi2_label_file_to_plugin_name(llfpath)
74 validate_plugin_name(tmp)
75 return tmp
77 def load(self):
78 if self.__loaded is True:
79 return
80 self.__loaded = True
81 c = self.configuration_class
82 self._configuration = c(
83 self.name, self.home,
84 app_class=self.app_class,
85 extra_daemon_class=self.extra_daemon_class,
86 dont_read_config_overrides=self._dont_read_config_overrides
87 )
88 self._layerapi2_layer_name = plugin_name_to_layerapi2_label(self.name)
89 self._load_format_version()
90 self._load_metadata()
91 self._load_version_release()
92 # self._load_files() is not included here for perfs reasons
94 def load_full(self):
95 self.load()
96 self.configuration.load()
98 def reload(self):
99 self.__loaded = False
100 self.load()
102 def _load_metadata(self):
103 if self.is_dev_linked:
104 self._is_installed = True
105 self._build_host = "unknown"
106 self._build_date = "unknown"
107 self._size = "unknown"
108 return
109 metadata_filepath = "%s/%s/.metadata.json" % (self.plugins_base_dir,
110 self.name)
111 self._is_installed = os.path.isfile(metadata_filepath)
112 if self._is_installed:
113 try:
114 with open(metadata_filepath, "r") as f:
115 c = f.read().strip()
116 except Exception as e:
117 raise BadPlugin("can't read %s file" % metadata_filepath,
118 original_exception=e)
119 try:
120 self._metadata = json.loads(c)
121 except Exception as e:
122 raise BadPlugin("can't decode %s file" % metadata_filepath,
123 original_exception=e)
124 self._build_host = self._metadata.get("build_host", "unknown")
125 self._build_date = self._metadata.get("build_date", "unknown")
126 self._size = self._metadata.get("size", "unknown")
128 def get_configuration_hash(self):
129 args = []
130 try:
131 with open("%s/.layerapi2_dependencies" % self.home, "r") as f:
132 args.append(f.read())
133 except Exception:
134 pass
135 for path in get_configuration_paths(
136 self.name, self.home) + ["/etc/metwork.config"]:
137 try:
138 with open(path, "r") as f:
139 args.append(f.read())
140 except Exception:
141 pass
142 return hash_generator(*args)
144 def get_plugin_env_dict(self, add_current_envs=True,
145 set_tmp_dir=True,
146 cache=False):
147 res = self._get_plugin_env_dict(add_current_envs=add_current_envs,
148 set_tmp_dir=set_tmp_dir,
149 cache=cache)
150 # this bloc is here and not inside _get_plugin_env_dict because
151 # PYTHONPATH shouldn't be cached (because it depends on loaded layers)
152 if self.configuration.add_plugin_dir_to_python_path:
153 old_python_path = os.environ.get("PYTHONPATH", None)
154 if old_python_path:
155 res["PYTHONPATH"] = self.home + ":" + old_python_path
156 else:
157 res["PYTHONPATH"] = self.home
158 return res
160 def _get_plugin_env_dict(self, add_current_envs=True,
161 set_tmp_dir=True,
162 cache=False):
163 if cache:
164 if not set_tmp_dir or not add_current_envs:
165 raise Exception(
166 "cache=True is not compatible with add_current_envs=False "
167 "or set_tmp_dir=False")
168 try:
169 with open("%s/.configuration_cache" % self.home, "rb") as f:
170 h, res = pickle.loads(f.read())
171 if h == self.get_configuration_hash():
172 res["%s_CURRENT_PLUGIN_CACHE" % MFMODULE] = "1"
173 tmpdir = res["TMPDIR"]
174 if tmpdir != "" and not os.path.exists(tmpdir):
175 mkdir_p(tmpdir, nodebug=True, nowarning=True)
176 return res
177 except Exception:
178 pass
179 lines = []
180 res = {}
181 try:
182 # FIXME: shoud be better to parse this file in layerapi2
183 with open("%s/.layerapi2_dependencies" % self.home, "r") as f:
184 lines = f.readlines()
185 except Exception:
186 pass
187 for line in lines:
188 tmp = line.strip()
189 if tmp.startswith('-'):
190 tmp = tmp[1:]
191 if tmp.startswith("plugin_"):
192 home = layerapi2_label_to_plugin_home(self.plugins_base_dir,
193 tmp)
194 if home is None:
195 continue
196 try:
197 p = Plugin(self.plugins_base_dir, home)
198 p.load()
199 except Exception:
200 continue
201 res.update(p.get_plugin_env_dict(add_current_envs=False))
202 env_var_dict = self.configuration.get_configuration_env_dict(
203 ignore_keys_starting_with="_")
204 res.update(env_var_dict)
205 if add_current_envs:
206 res.update(get_current_envs(self.name, self.home))
207 if set_tmp_dir:
208 tmpdir = os.path.join(MFMODULE_RUNTIME_HOME, "tmp", self.name)
209 if mkdir_p(tmpdir, nodebug=True, nowarning=True):
210 res["TMPDIR"] = tmpdir
211 if cache:
212 h = self.get_configuration_hash()
213 tmpname = "%s/.configuration_cache.%s" % \
214 (self.home, get_unique_hexa_identifier())
215 with open(tmpname, "wb") as f:
216 f.write(pickle.dumps([h, res]))
217 os.rename(tmpname, "%s/.configuration_cache" % self.home)
218 Path('%s/.configuration_cache' % self.home).touch()
219 return res
221 def plugin_env_context(self, **kwargs):
222 return PluginEnvContextManager(self.get_plugin_env_dict(**kwargs))
224 def _load_version_release(self):
225 if not self._is_installed:
226 # the plugin is not installed, let's read version in configuration
227 self._version = self.configuration.version
228 if self.configuration.release:
229 self._release = self.configuration.release
230 else:
231 self._release = "1"
232 return
233 # the plugin is installed
234 if self.is_dev_linked:
235 # this is a devlink
236 self._version = "dev_link"
237 self._release = "dev_link"
238 return
239 self._version = self._metadata["version"]
240 self._release = self._metadata["release"]
242 def _load_format_version(self):
243 pfv = os.path.join(self.home, ".plugin_format_version")
244 if not os.path.isfile(pfv):
245 raise BadPlugin("%s is missing => this is probably an old and "
246 "incompatible plugin => please migrate it!" % pfv)
247 try:
248 with open("%s/.plugin_format_version" % self.home, "r") as f:
249 c = f.read().strip()
250 tmp = c.split('.')
251 if len(tmp) < 3:
252 raise Exception()
253 except Exception:
254 raise BadPlugin("bad %s/.plugin_format_version format!" %
255 self.home)
256 res = []
257 for t in tmp:
258 try:
259 res.append(int(t))
260 except Exception:
261 res.append(9999)
262 self._format_version = res[0:3]
264 def print_dangerous_state(self):
265 res = BashWrapper("_plugins.is_dangerous %s" % (self.name,))
266 if res and res.stdout and len(res.stdout) > 0:
267 print(res.stdout)
269 def get_hash(self):
270 sid = ", ".join([self.build_host, self.build_date, self.size,
271 self.version, self.release])
272 return hashlib.md5(sid.encode('utf8')).hexdigest()
274 def repackage(self):
275 self.load()
276 tmpdir = os.path.join(MFMODULE_RUNTIME_HOME, "tmp",
277 "plugin_%s" % get_unique_hexa_identifier())
278 mkdir_p_or_die(tmpdir)
279 shutil.copytree(self.home, os.path.join(tmpdir, "metwork_plugin"),
280 symlinks=True)
282 def build(self):
283 self.load()
284 pwd = os.getcwd()
285 tmpdir = os.path.join(MFMODULE_RUNTIME_HOME, "tmp",
286 "plugin_%s" % get_unique_hexa_identifier())
287 filename = f"{self.name}-{self.version}-{self.release}." \
288 f"metwork.{MFMODULE_LOWERCASE}.plugin"
289 mkdir_p_or_die(tmpdir)
290 shutil.copytree(self.home, os.path.join(tmpdir, "metwork_plugin"),
291 symlinks=True)
292 matches = None
293 ignore_filepath = os.path.join(self.home, ".releaseignore")
294 if os.path.isfile(ignore_filepath):
295 try:
296 matches = parse_gitignore(ignore_filepath)
297 except Exception as e:
298 raise BadPlugin("bad %s file" % ignore_filepath,
299 original_exception=e)
300 root = os.path.join(tmpdir, "metwork_plugin")
301 if matches is not None:
302 for r, d, f in os.walk(root, topdown=False):
303 for fle in f:
304 full_path = os.path.join(r, fle)
305 path = self.home + full_path[len(root):]
306 if matches(path):
307 try:
308 os.unlink(full_path)
309 except Exception:
310 pass
311 for flder in d:
312 full_path = os.path.join(r, flder)
313 path = self.home + full_path[len(root):]
314 if matches(path) and not os.listdir(full_path):
315 shutil.rmtree(full_path, ignore_errors=True)
316 files = []
317 total_size = 0
318 for r, d, f in os.walk(os.path.join(tmpdir, "metwork_plugin")):
319 for fle in f:
320 path = os.path.join(r, fle)
321 files.append("metwork_plugin/" + path[len(root) + 1:])
322 if not os.path.islink(path):
323 total_size = total_size + os.path.getsize(path)
324 with open("%s/metwork_plugin/.files.json" % tmpdir, "w") as f:
325 f.write(json.dumps(files, indent=4))
327 # utcnow() is deprecated and should be replaced by now(datetime.UTC)
328 # (for python >= 3.11)
329 try:
330 build_date = datetime.now(timezone.utc).replace(
331 tzinfo=None).isoformat()[0:19] + 'Z'
332 except Exception:
333 build_date = datetime.utcnow().isoformat()[0:19] + 'Z'
335 metadata = {
336 "version": self.version,
337 "release": self.release,
338 "build_host": BUID_HOST,
339 "build_date": build_date,
340 "size": str(total_size),
341 "summary": self.configuration.summary,
342 "license": self.configuration.license,
343 "packager": self.configuration.packager,
344 "vendor": self.configuration.vendor,
345 "url": self.configuration.url
346 }
347 with open("%s/metwork_plugin/.metadata.json" % tmpdir, "w") as f:
348 f.write(json.dumps(metadata, indent=4))
349 plugin_path = os.path.abspath(f"{pwd}/{filename}")
350 cmd = f"cd {tmpdir} && tar -cvf plugin.tar metwork_plugin && " \
351 f"gzip -f plugin.tar && " \
352 f"mv plugin.tar.gz {plugin_path}"
353 BashWrapperOrRaise(cmd, CantBuildPlugin)
354 if not os.path.isfile(plugin_path):
355 raise CantBuildPlugin("can't find plugin file: %s" % plugin_path)
356 shutil.rmtree(tmpdir, True)
357 return plugin_path
359 def _load_files(self):
360 if self._files is not None:
361 return
362 if self.is_dev_linked:
363 self._files = []
364 return
365 if not self.is_installed:
366 self._files = []
367 return
368 filepath = "%s/%s/.files.json" % (self.plugins_base_dir, self.name)
369 if not os.path.isfile(filepath):
370 raise BadPlugin("%s is missing" % filepath)
371 try:
372 with open(filepath, "r") as f:
373 c = f.read().strip()
374 except Exception as e:
375 raise BadPlugin("can't read %s file" % filepath,
376 original_exception=e)
377 try:
378 self._files = json.loads(c)
379 except Exception as e:
380 raise BadPlugin("can't decode %s file" % filepath,
381 original_exception=e)
383 @property
384 def configuration(self):
385 self.load()
386 return self._configuration
388 @property
389 def layerapi2_layer_name(self):
390 self.load()
391 return self._layerapi2_layer_name
393 @property
394 def format_version(self):
395 self.load()
396 return self._format_version
398 @property
399 def version(self):
400 self.load()
401 return self._version
403 @property
404 def release(self):
405 self.load()
406 return self._release
408 @property
409 def build_host(self):
410 self.load()
411 return self._build_host
413 @property
414 def build_date(self):
415 self.load()
416 return self._build_date
418 @property
419 def size(self):
420 self.load()
421 return self._size
423 @property
424 def is_installed(self):
425 self.load()
426 return self._is_installed
428 @property
429 def files(self):
430 self.load()
431 self._load_files() # not included in load() for perfs reasons
432 return self._files