Configuration file
This is the default configuration file for plugin template: ungzip
###################################
##### #####
##### GENERAL SECTION #####
##### #####
###################################
# (plugin metadatas)
[general]
# Notes about the name of the plugin:
# - the name of the plugin is given by the content of .layerapi2_label
# in this directory (format: plugin_{name of the plugin}@mfdata)
# - the old "name" key in this file is not used anymore
# Version of the plugin (X.Y.Z)
# If the value is {{MFMODULE_VERSION}},
# the current module version is used
_version={{MFMODULE_VERSION}}
# Release of the plugin
_release=1
# Summary (one line) of the goal of the plugin
_summary=one line summary about your plugin
# License of the plugin
_license=Proprietary
# URL of the plugin homepage
_url=http://yourpluginhomepage
# Name of the maintainer of the plugin on the form "Firstname FAMILYNAME <email>"
_maintainer=Firstname FAMILYNAME <email>
# Vendor of the plugin
_vendor=MetWork
#################################
##### #####
##### STEP SECTIONS #####
##### #####
#################################
# (one section [step_xxxxx] for each step)
# You can have several blocks [step_xxxx] if your plugin
# contains several steps.
# If you have only one step in your plugin, please use "main" as your
# step name in [step_xxxxx]. So just one [step_name] section in this case.
[step_main]
# Command (and args) to be executed
# Notes :
# - null means no command to execute
# - this command can be launched several times (see workers/numprocesses)
# - the command must pop redis queue {{MFDATA_CURRENT_STEP_QUEUE}}
# (dynamically generated), so it must include {MFDATA_CURRENT_STEP_QUEUE} in its args
# You can use following placeholders :
# - {{MFDATA_CURRENT_PLUGIN_NAME}}: replaced by the plugin name
# - {{MFDATA_CURRENT_PLUGIN_DIR}}: replaced by the plugin
# directory fullpath.
# - {MFDATA_CURRENT_STEP_NAME}: replaced by the step name
# - {MFDATA_CURRENT_STEP_QUEUE}: replaced by the name of the queue
# for the plugin/step on redis
# - {MFDATA_CURRENT_STEP_DIR}: replaced by the step directory
# - {CUSTOM_...}: replaced by the corresponding configuration value under
# the [custom] section of this file (example: {CUSTOM_FOO} will be
# replaced by the value of the configuration key "foo" under [custom]
# section.
# - {{ENV_VAR}}: replaced by the corresponding env var.
# - {STANDARD_STEP_ARGUMENTS}: replaced by various but standard arguments
# for every step (must be always here at the end)
# - it's not the command called for each incoming file, it's a daemon implemented
# with Acquisition framework in Python; if you want to execute a binary command
# for each incoming file, please bootstrap a "fork" plugin
_cmd_and_args=ungzip_step --dest-dir={CUSTOM_DEST_DIR} {STANDARD_STEP_ARGUMENTS}
# Number of retry in case of errors (before applying the choosen failure policy)
# 0 => no retry
# x => maximum x retries
retry_total = 0
# The minimum number of seconds (can be a float) to wait before retrying
# (see retry_backoff for details about the algorithm)
retry_min_wait=10
# The maximum number of seconds (can be a float) to wait before retrying
# (see retry_backoff for details about the algorithm)
retry_max_wait=120
# Before each retry, we wait:
# retry_min_wait + retry_backoff * ( 2 ** (retry_attempt_number-1) )
# So with a backoff == 0 (default), we wait exactly retry_min_wait seconds
# before each retry.
# With a backoff == 1, we wait retry_min_wait + 1 seconds for the first retry
# we wait retry_min_wait + 2 seconds for the second retry
# we wait retry_min_wait + 4 seconds for the third retry
# [...] note that the waiting time is capped by retry_max_wait
# (retry_backoff can be a float)
retry_backoff=0
# Failure policy : keep (default), delete or move
failure_policy = keep
# Destination directory when failure_policy is move (mandatory in this case)
# (if failure_policy != move, this configuration key is not used)
# must be something like: plugin_name/step_name
failure_policy_move_dest_dir =
# If set then the process will be restarted sometime after max_age and
# max_age + random(0, max_age) seconds.
# 0 => disable this feature
max_age = 3600
# Split stdout/stderr logs into different files?
# AUTO => the global conf MFDATA_LOG_TRY_TO_SPLIT_STDOUT_STDERR is used
# (default: don't split)
# 1 => split
# 0 => don't split
log_split_stdout_stderr=AUTO
# Split logs of multiple workers into different log files?
# AUTO => the global conf MFDATA_LOG_TRY_TO_SPLIT_MULTIPLE_WORKERS is used
# (default: don't split)
# 1 => split
# 0 => don't split
log_split_multiple_workers=AUTO
# The number of processes to run for this step
# note: you can use {{MFHARDWARE_NUMBER_OF_CPU_CORES_MULTIPLIED_BY_2}} value
# to avoid a kind of hardware automatic value (see "env |grep HARDWARE" as mfdata
# to find other automatic values)
numprocesses = 1
# if you set debug=1, numprocesses will be forced to 1
# and we will try to set all debug options and features
# (max_age will also be forced to 0 and mflog minimal level will be set to DEBUG)
# DON'T USE IT ON PRODUCTION!!!
debug=0
# The number of seconds to wait for a step to terminate gracefully
# before killing it. When stopping a process, we first send it a TERM signal.
# A step may catch this signal to perform clean up operations before exiting.
# If the worker is still active after {timeout} seconds, we send it a
# KILL signal. It is not possible to catch a KILL signal, so the worker will stop.
# If you use the standard Acquisition framework to implement your step, the
# TERM signal is handled like this: "we don't process files anymore but we
# try to end with the current processed file before stopping". So the
# timeout must by greater than the maximum processing time of one file.
# (must be set and >0)
timeout=600
# !!! ADVANCED SETTINGS !!!
# Coma separated list of watched directories (relative to MFDATA_DATA_IN_DIR)
# You should keep {MFDATA_CURRENT_STEP_DIR} here unless you know exactly
# what you are doing.
# If several plugins watch the same directory, they will compete to process
# files. If you want to copy files to multiple plugins, use a copy or a
# switch plugin.
# This parameter can be useful only if you don't want to use
# default guess_file_type and switch plugins (advanced use)
watched_directories={MFDATA_CURRENT_STEP_DIR}
# resource limit for each step process
# rlimit_as => maximum area (in bytes) of address space which may be taken
# by the process.
# rlimit_nofile => maximum number of open file descriptors for the current
# process.
# rlimit_stack => maximum size (in bytes) of the call stack for the current
# process.
# This only affects the stack of the main thread in a multi-threaded process.
# rlimit_fsize => maximum size of a file which the process may create.
# (empty value means no limit)
rlimit_as = 1000000000
rlimit_nofile = 1000
rlimit_stack = 10000000
rlimit_fsize = 10000000000
# !!! ADVANCED SETTINGS !!!
# You can add extra daemons which will be launched within your plugin
# by providing configuration blocks [extra_daemon_*]
# You have to provide a command to daemonize (the command must run in
# foreground and not daemonize by itself)
# [extra_daemon_foo]
# _cmd_and_args = /your/foreground/command command_arg1 command_arg2
# numprocesses=1
# graceful_timeout = 30
# rlimit_as = 1000000000
# rlimit_nofile = 1000
# rlimit_stack = 10000000
# rlimit_fsize = 100000000
# log_split_stdout_stderr=AUTO
# log_split_multiple_workers=AUTO
# max_age=0
####################################################################
##### #####
##### SWITCH RULES FRAGMENT (FOR THIS PLUGIN) #####
##### (only if you use the default system switch plugin) #####
##### #####
####################################################################
#
# You can have several "rules blocks":
# [switch_rules:{rule_type}:{rule_param1,rule_param2}]
#
# Each "rules block" define a rule type and some rule parameters.
# Under a "rules block" you can have one or several "switch rules".
#
# A "switch rule" is a line like:
# pattern = plugin_name1/step_name1, plugin_name2/step_name2*
#
# So, for a given pattern, you can have one or several copy actions (separated
# by coma). Each action means copy (or hardlink) the incoming file to the
# given plugin/step.
#
# (if a step name ends with a "*", it means that we can use hardlinking instead
# of copying => it's better for performances but target step must not alter
# the incoming file in any way so please DO NOT ADD THE STAR SIGN IF YOU ARE
# NOT SURE)
#
# All "switch rules" are evaluated in the context of their "rule block". If
# the pattern match (in this context), collected actions are APPENDED.
#
# See FIXME for plenty details and examples.
# Example: copy all incoming files with original_basename starting with A to
# the current plugin in "main" step
# [switch_rules:fnmatch:first.core.original_basename]
# A* = main
# Example: copy all incoming files to the current plugin in "main" step
# [switch_rules:alwaystrue]
# * = main
# Example: copy all incoming files from the foo dirname to the current plugin
# in "main" step
# [switch_rules:equal:first.core.original_dirname]
# foo = main
# Example: for complex routing rules, you can use a python function
# (here: my_function() in foo.py under the plugin root)
# [switch_rules:python]
# foo.my_function = main
#
[switch_rules:fnmatch:latest.guess_file_type.main.system_magic]
gzip compressed data* = {{MFDATA_CURRENT_PLUGIN_NAME}}/main
[custom]
# Destination plugin/step of the ungzipped file
#
# Use syntax like: plugin_name/step_name
dest_dir = guess_file_type/main