Coverage for hooks/nrpe_helpers.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Nrpe helpers module."""
2import glob
3import ipaddress
4import os
5import socket
6import subprocess
8from charmhelpers.core import hookenv
9from charmhelpers.core.host import is_container
10from charmhelpers.core.services import helpers
12import yaml
15NETLINKS_ERROR = False
18class InvalidCustomCheckException(Exception):
19 """Custom exception for Invalid nrpe check."""
21 pass
24class Monitors(dict):
25 """List of checks that a remote Nagios can query."""
27 def __init__(self, version="0.3"):
28 """Build monitors structure."""
29 self["monitors"] = {"remote": {"nrpe": {}}}
30 self["version"] = version
32 def add_monitors(self, mdict, monitor_label="default"):
33 """Add monitors passed in mdict."""
34 if not mdict or not mdict.get("monitors"):
35 return
37 for checktype in mdict["monitors"].get("remote", []):
38 check_details = mdict["monitors"]["remote"][checktype]
39 if self["monitors"]["remote"].get(checktype):
40 self["monitors"]["remote"][checktype].update(check_details)
41 else:
42 self["monitors"]["remote"][checktype] = check_details
44 for checktype in mdict["monitors"].get("local", []):
45 check_details = self.convert_local_checks(
46 mdict["monitors"]["local"],
47 monitor_label,
48 )
49 self["monitors"]["remote"]["nrpe"].update(check_details)
51 def add_nrpe_check(self, check_name, command):
52 """Add nrpe check to remote monitors."""
53 self["monitors"]["remote"]["nrpe"][check_name] = command
55 def convert_local_checks(self, monitors, monitor_src):
56 """Convert check from local checks to remote nrpe checks.
58 monitors -- monitor dict
59 monitor_src -- Monitor source principal, subordinate or user
60 """
61 mons = {}
62 for checktype in monitors.keys():
63 for checkname in monitors[checktype]:
64 try:
65 check_def = NRPECheckCtxt(
66 checktype,
67 monitors[checktype][checkname],
68 monitor_src,
69 )
70 mons[check_def["cmd_name"]] = {"command": check_def["cmd_name"]}
71 except InvalidCustomCheckException as e:
72 hookenv.log(
73 "Error encountered configuring local check "
74 '"{check}": {err}'.format(check=checkname, err=str(e)),
75 hookenv.ERROR,
76 )
77 return mons
80def get_ingress_address(binding, external=False):
81 """Get ingress IP address for a binding.
83 Returns a local IP address for incoming requests to NRPE.
85 :param binding: name of the binding, e.g. 'monitors'
86 :param external: bool, if True return the public address if charm config requests
87 otherwise return the local address which would be used for incoming
88 nrpe requests.
89 """
90 # using network-get to retrieve the address details if available.
91 hookenv.log("Getting ingress IP address for binding %s" % binding)
92 if hookenv.config("nagios_address_type").lower() == "public" and external:
93 return hookenv.unit_get("public-address")
95 ip_address = None
96 try:
97 network_info = hookenv.network_get(binding)
98 if network_info is not None and "ingress-addresses" in network_info:
99 try:
100 ip_address = network_info["bind-addresses"][0]["addresses"][0][
101 "address"
102 ]
103 hookenv.log("Using ingress-addresses, found %s" % ip_address)
104 except KeyError:
105 hookenv.log("Using primary-addresses")
106 ip_address = hookenv.network_get_primary_address(binding)
108 except (NotImplementedError, FileNotFoundError) as e:
109 hookenv.log(
110 "Unable to determine inbound IP address for binding {} with {}".format(
111 binding, e
112 ),
113 level=hookenv.ERROR,
114 )
116 return ip_address
119class MonitorsRelation(helpers.RelationContext):
120 """Define a monitors relation."""
122 name = "monitors"
123 interface = "monitors"
125 def __init__(self, *args, **kwargs):
126 """Build superclass and principal relation."""
127 self.principal_relation = PrincipalRelation()
128 super(MonitorsRelation, self).__init__(*args, **kwargs)
130 def is_ready(self):
131 """Return true if the principal relation is ready."""
132 return self.principal_relation.is_ready()
134 def get_subordinate_monitors(self):
135 """Return default monitors defined by this charm."""
136 monitors = Monitors()
137 for check in SubordinateCheckDefinitions()["checks"]:
138 if check["cmd_params"]:
139 monitors.add_nrpe_check(check["cmd_name"], check["cmd_name"])
140 return monitors
142 def get_user_defined_monitors(self):
143 """Return monitors defined by monitors config option."""
144 monitors = Monitors()
145 monitors.add_monitors(yaml.safe_load(hookenv.config("monitors")), "user")
146 return monitors
148 def get_principal_monitors(self):
149 """Return monitors passed by relation with principal."""
150 return self.principal_relation.get_monitors()
152 def get_monitor_dicts(self):
153 """Return all monitor dicts."""
154 monitor_dicts = {
155 "principal": self.get_principal_monitors(),
156 "subordinate": self.get_subordinate_monitors(),
157 "user": self.get_user_defined_monitors(),
158 }
159 return monitor_dicts
161 def get_monitors(self):
162 """Return monitor dict.
164 All monitors merged together and local
165 monitors converted to remote nrpe checks.
166 """
167 all_monitors = Monitors()
168 monitors = [
169 self.get_principal_monitors(),
170 self.get_subordinate_monitors(),
171 self.get_user_defined_monitors(),
172 ]
173 for mon in monitors:
174 all_monitors.add_monitors(mon)
175 return all_monitors
177 def egress_subnets(self, relation_data):
178 """Return egress subnets.
180 This behaves the same as charmhelpers.core.hookenv.egress_subnets().
181 If it can't determine the egress subnets it will fall back to
182 ingress-address or finally private-address.
183 """
184 if "egress-subnets" in relation_data:
185 return relation_data["egress-subnets"]
186 if "ingress-address" in relation_data:
187 return relation_data["ingress-address"]
188 return relation_data["private-address"]
190 def get_data(self):
191 """Get relation data."""
192 super(MonitorsRelation, self).get_data()
193 if not hookenv.relation_ids(self.name):
194 return
195 # self['monitors'] comes from the superclass helpers.RelationContext
196 # and contains relation data for each 'monitors' relation (to/from
197 # Nagios).
198 subnets = [self.egress_subnets(info) for info in self["monitors"]]
199 self["monitor_allowed_hosts"] = ",".join(subnets)
201 def provide_data(self):
202 """Return relation info."""
203 # get the address to send to Nagios for host definition
204 address = get_ingress_address("monitors", external=True)
206 relation_info = {
207 "target-id": self.principal_relation.nagios_hostname(),
208 "monitors": self.get_monitors(),
209 "private-address": address,
210 "ingress-address": address,
211 "target-address": address,
212 "machine_id": os.environ["JUJU_MACHINE_ID"],
213 "model_id": hookenv.model_uuid(),
214 }
215 return relation_info
218class PrincipalRelation(helpers.RelationContext):
219 """Define a principal relation."""
221 def __init__(self, *args, **kwargs):
222 """Set name and interface."""
223 if hookenv.relations_of_type("nrpe-external-master"):
224 self.name = "nrpe-external-master"
225 self.interface = "nrpe-external-master"
226 elif hookenv.relations_of_type("general-info"):
227 self.name = "general-info"
228 self.interface = "juju-info"
229 elif hookenv.relations_of_type("local-monitors"):
230 self.name = "local-monitors"
231 self.interface = "local-monitors"
232 super(PrincipalRelation, self).__init__(*args, **kwargs)
234 def is_ready(self):
235 """Return true if the relation is connected."""
236 if self.name not in self:
237 return False
238 return "__unit__" in self[self.name][0]
240 def nagios_hostname(self):
241 """Return the string that nagios will use to identify this host."""
242 host_context = hookenv.config("nagios_host_context")
243 if host_context:
244 host_context += "-"
245 hostname_type = hookenv.config("nagios_hostname_type")
247 # Detect bare metal hosts
248 if hostname_type == "auto":
249 is_metal = "none" in subprocess.getoutput("/usr/bin/systemd-detect-virt")
250 if is_metal:
251 hostname_type = "host"
252 else:
253 hostname_type = "unit"
255 if hostname_type == "host" or not self.is_ready():
256 nagios_hostname = "{}{}".format(host_context, socket.gethostname())
257 return nagios_hostname
258 else:
259 principal_unitname = hookenv.principal_unit()
260 # Fallback to using "primary" if it exists.
261 if not principal_unitname:
262 for relunit in self[self.name]:
263 if relunit.get("primary", "False").lower() == "true":
264 principal_unitname = relunit["__unit__"]
265 break
266 nagios_hostname = "{}{}".format(host_context, principal_unitname)
267 nagios_hostname = nagios_hostname.replace("/", "-")
268 return nagios_hostname
270 def get_monitors(self):
271 """Return monitors passed by services on the self.interface relation."""
272 if not self.is_ready():
273 return
274 monitors = Monitors()
275 for rel in self[self.name]:
276 if rel.get("monitors"):
277 monitors.add_monitors(yaml.load(rel["monitors"]), "principal")
278 return monitors
280 def provide_data(self):
281 """Return nagios hostname and nagios host context."""
282 # Provide this data to principals because get_nagios_hostname expects
283 # them in charmhelpers/contrib/charmsupport/nrpe when writing principal
284 # service__* files
285 return {
286 "nagios_hostname": self.nagios_hostname(),
287 "nagios_host_context": hookenv.config("nagios_host_context"),
288 }
291class NagiosInfo(dict):
292 """Define a NagiosInfo dict."""
294 def __init__(self):
295 """Set principal relation and dict values."""
296 self.principal_relation = PrincipalRelation()
297 self["external_nagios_master"] = "127.0.0.1"
298 if hookenv.config("nagios_master") != "None":
299 self["external_nagios_master"] = "{},{}".format(
300 self["external_nagios_master"], hookenv.config("nagios_master")
301 )
302 self["nagios_hostname"] = self.principal_relation.nagios_hostname()
304 # export_host.cfg.tmpl host definition for Nagios
305 self["nagios_ipaddress"] = get_ingress_address("monitors", external=True)
306 # Address configured for NRPE to listen on
307 self["nrpe_ipaddress"] = get_ingress_address("monitors")
309 self["dont_blame_nrpe"] = "1" if hookenv.config("dont_blame_nrpe") else "0"
310 self["debug"] = "1" if hookenv.config("debug") else "0"
313class RsyncEnabled(helpers.RelationContext):
314 """Define a relation context for rsync enabled relation."""
316 def __init__(self):
317 """Set export_nagios_definitions."""
318 self["export_nagios_definitions"] = hookenv.config("export_nagios_definitions")
319 if (
320 hookenv.config("nagios_master")
321 and hookenv.config("nagios_master") != "None"
322 ):
323 self["export_nagios_definitions"] = True
325 def is_ready(self):
326 """Return true if relation is ready."""
327 return self["export_nagios_definitions"]
330class NRPECheckCtxt(dict):
331 """Convert a local monitor definition.
333 Create a dict needed for writing the nrpe check definition.
334 """
336 def __init__(self, checktype, check_opts, monitor_src):
337 """Set dict values."""
338 plugin_path = "/usr/lib/nagios/plugins"
339 if checktype == "procrunning":
340 self["cmd_exec"] = plugin_path + "/check_procs"
341 self["description"] = "Check process {executable} is running".format(
342 **check_opts
343 )
344 self["cmd_name"] = "check_proc_" + check_opts["executable"]
345 self["cmd_params"] = "-w {min} -c {max} -C {executable}".format(
346 **check_opts
347 )
348 elif checktype == "processcount":
349 self["cmd_exec"] = plugin_path + "/check_procs"
350 self["description"] = "Check process count"
351 self["cmd_name"] = "check_proc_principal"
352 if "min" in check_opts:
353 self["cmd_params"] = "-w {min} -c {max}".format(**check_opts)
354 else:
355 self["cmd_params"] = "-c {max}".format(**check_opts)
356 elif checktype == "disk":
357 self["cmd_exec"] = plugin_path + "/check_disk"
358 self["description"] = "Check disk usage " + check_opts["path"].replace(
359 "/", "_"
360 )
361 self["cmd_name"] = "check_disk_principal"
362 self["cmd_params"] = "-w 20 -c 10 -p " + check_opts["path"]
363 elif checktype == "custom":
364 custom_path = check_opts.get("plugin_path", plugin_path)
365 if not custom_path.startswith(os.path.sep):
366 custom_path = os.path.join(os.path.sep, custom_path)
367 if not os.path.isdir(custom_path):
368 raise InvalidCustomCheckException(
369 'Specified plugin_path "{}" does not exist or is not a '
370 "directory.".format(custom_path)
371 )
372 check = check_opts["check"]
373 self["cmd_exec"] = os.path.join(custom_path, check)
374 self["description"] = check_opts.get("desc", "Check %s" % check)
375 self["cmd_name"] = check
376 self["cmd_params"] = check_opts.get("params", "") or ""
377 self["description"] += " ({})".format(monitor_src)
378 self["cmd_name"] += "_" + monitor_src
381class SubordinateCheckDefinitions(dict):
382 """Return dict of checks the charm configures."""
384 def __init__(self):
385 """Set dict values."""
386 self.procs = self.proc_count()
387 load_thresholds = self._get_load_thresholds()
388 proc_thresholds = self._get_proc_thresholds()
389 disk_root_thresholds = self._get_disk_root_thresholds()
391 pkg_plugin_dir = "/usr/lib/nagios/plugins/"
392 local_plugin_dir = "/usr/local/lib/nagios/plugins/"
393 checks = [
394 {
395 "description": "Number of Zombie processes",
396 "cmd_name": "check_zombie_procs",
397 "cmd_exec": pkg_plugin_dir + "check_procs",
398 "cmd_params": hookenv.config("zombies"),
399 },
400 {
401 "description": "Number of processes",
402 "cmd_name": "check_total_procs",
403 "cmd_exec": pkg_plugin_dir + "check_procs",
404 "cmd_params": proc_thresholds,
405 },
406 {
407 "description": "Number of Users",
408 "cmd_name": "check_users",
409 "cmd_exec": pkg_plugin_dir + "check_users",
410 "cmd_params": hookenv.config("users"),
411 },
412 {
413 "description": "Connnection tracking table",
414 "cmd_name": "check_conntrack",
415 "cmd_exec": local_plugin_dir + "check_conntrack.sh",
416 "cmd_params": hookenv.config("conntrack"),
417 },
418 ]
420 if not is_container():
421 checks.extend(
422 [
423 {
424 "description": "Root disk",
425 "cmd_name": "check_disk_root",
426 "cmd_exec": pkg_plugin_dir + "check_disk",
427 "cmd_params": disk_root_thresholds,
428 },
429 {
430 "description": "System Load",
431 "cmd_name": "check_load",
432 "cmd_exec": pkg_plugin_dir + "check_load",
433 "cmd_params": load_thresholds,
434 },
435 {
436 "description": "Swap",
437 "cmd_name": "check_swap",
438 "cmd_exec": pkg_plugin_dir + "check_swap",
439 "cmd_params": hookenv.config("swap").strip(),
440 },
441 # Note: check_swap_activity *must* be listed after check_swap, else
442 # check_swap_activity will be removed during installation of
443 # check_swap.
444 {
445 "description": "Swap Activity",
446 "cmd_name": "check_swap_activity",
447 "cmd_exec": local_plugin_dir + "check_swap_activity",
448 "cmd_params": hookenv.config("swap_activity"),
449 },
450 {
451 "description": "Memory",
452 "cmd_name": "check_mem",
453 "cmd_exec": local_plugin_dir + "check_mem.pl",
454 "cmd_params": hookenv.config("mem"),
455 },
456 {
457 "description": "XFS Errors",
458 "cmd_name": "check_xfs_errors",
459 "cmd_exec": local_plugin_dir + "check_xfs_errors.py",
460 "cmd_params": hookenv.config("xfs_errors"),
461 },
462 {
463 "description": "ARP cache entries",
464 "cmd_name": "check_arp_cache",
465 "cmd_exec": os.path.join(
466 local_plugin_dir, "check_arp_cache.py"
467 ),
468 "cmd_params": "-w 60 -c 80",
469 },
470 ]
471 )
473 ro_filesystem_excludes = hookenv.config("ro_filesystem_excludes")
474 if ro_filesystem_excludes == "":
475 # specify cmd_params = '' to disable/remove the check from nrpe
476 check_ro_filesystem = {
477 "description": "Readonly filesystems",
478 "cmd_name": "check_ro_filesystem",
479 "cmd_exec": os.path.join(
480 local_plugin_dir, "check_ro_filesystem.py"
481 ),
482 "cmd_params": "",
483 }
484 else:
485 check_ro_filesystem = {
486 "description": "Readonly filesystems",
487 "cmd_name": "check_ro_filesystem",
488 "cmd_exec": os.path.join(
489 local_plugin_dir, "check_ro_filesystem.py"
490 ),
491 "cmd_params": "-e {}".format(
492 hookenv.config("ro_filesystem_excludes")
493 ),
494 }
495 checks.append(check_ro_filesystem)
497 if hookenv.config("lacp_bonds").strip():
498 for bond_iface in hookenv.config("lacp_bonds").strip().split():
499 if os.path.exists("/sys/class/net/{}".format(bond_iface)):
500 description = "LACP Check {}".format(bond_iface)
501 cmd_name = "check_lacp_{}".format(bond_iface)
502 cmd_exec = local_plugin_dir + "check_lacp_bond.py"
503 cmd_params = "-i {}".format(bond_iface)
504 lacp_check = {
505 "description": description,
506 "cmd_name": cmd_name,
507 "cmd_exec": cmd_exec,
508 "cmd_params": cmd_params,
509 }
510 checks.append(lacp_check)
512 if hookenv.config("netlinks"):
513 ifaces = yaml.safe_load(hookenv.config("netlinks"))
514 cmd_exec = local_plugin_dir + "check_netlinks.py"
515 if hookenv.config("netlinks_skip_unfound_ifaces"):
516 cmd_exec += " --skip-unfound-ifaces"
517 d_ifaces = self.parse_netlinks(ifaces)
518 for iface in d_ifaces:
519 description = "Netlinks status ({})".format(iface)
520 cmd_name = "check_netlinks_{}".format(iface)
521 cmd_params = d_ifaces[iface]
522 netlink_check = {
523 "description": description,
524 "cmd_name": cmd_name,
525 "cmd_exec": cmd_exec,
526 "cmd_params": cmd_params,
527 }
528 checks.append(netlink_check)
530 # Checking if CPU governor is supported by the system and add nrpe check
531 cpu_governor_paths = "/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"
532 cpu_governor_supported = glob.glob(cpu_governor_paths)
533 requested_cpu_governor = hookenv.relation_get("requested_cpu_governor")
534 cpu_governor_config = hookenv.config("cpu_governor")
535 wanted_cpu_governor = cpu_governor_config or requested_cpu_governor
536 if wanted_cpu_governor and cpu_governor_supported:
537 description = "Check CPU governor scaler"
538 cmd_name = "check_cpu_governor"
539 cmd_exec = local_plugin_dir + "check_cpu_governor.py"
540 cmd_params = "--governor {}".format(wanted_cpu_governor)
541 cpu_governor_check = {
542 "description": description,
543 "cmd_name": cmd_name,
544 "cmd_exec": cmd_exec,
545 "cmd_params": cmd_params,
546 }
547 checks.append(cpu_governor_check)
549 self["checks"] = []
550 sub_postfix = str(hookenv.config("sub_postfix"))
551 # Automatically use _sub for checks shipped on a unit with the nagios
552 # charm. Mostly for backwards compatibility.
553 principal_unit = hookenv.principal_unit()
554 if sub_postfix == "" and principal_unit:
555 md = hookenv._metadata_unit(principal_unit)
556 if md and md.pop("name", None) == "nagios":
557 sub_postfix = "_sub"
558 nrpe_config_sub_tmpl = "/etc/nagios/nrpe.d/{}_*.cfg"
559 nrpe_config_tmpl = "/etc/nagios/nrpe.d/{}.cfg"
560 for check in checks:
561 # This can be used to clean up old files before rendering the new
562 # ones
563 nrpe_configfiles_sub = nrpe_config_sub_tmpl.format(check["cmd_name"])
564 nrpe_configfiles = nrpe_config_tmpl.format(check["cmd_name"])
565 check["matching_files"] = glob.glob(nrpe_configfiles_sub)
566 check["matching_files"].extend(glob.glob(nrpe_configfiles))
567 check["description"] += " (sub)"
568 check["cmd_name"] += sub_postfix
569 self["checks"].append(check)
571 def _get_proc_thresholds(self):
572 """Return suitable processor thresholds."""
573 if hookenv.config("procs") == "auto":
574 proc_thresholds = "-k -w {} -c {}".format(
575 25 * self.procs + 100, 50 * self.procs + 100
576 )
577 else:
578 proc_thresholds = hookenv.config("procs")
579 return proc_thresholds
581 def _get_load_thresholds(self):
582 """Return suitable load thresholds."""
583 if hookenv.config("load") == "auto":
584 # Give 1min load alerts higher thresholds than 15 min load alerts
585 warn_multipliers = (4, 2, 1)
586 crit_multipliers = (8, 4, 2)
587 load_thresholds = ("-w %s -c %s") % (
588 ",".join([str(m * self.procs) for m in warn_multipliers]),
589 ",".join([str(m * self.procs) for m in crit_multipliers]),
590 )
591 else:
592 load_thresholds = hookenv.config("load")
593 return load_thresholds
595 def _get_disk_root_thresholds(self):
596 """Return suitable disk thresholds."""
597 if hookenv.config("disk_root"):
598 disk_root_thresholds = hookenv.config("disk_root") + " -p / "
599 else:
600 disk_root_thresholds = ""
601 return disk_root_thresholds
603 def proc_count(self):
604 """Return number number of processing units."""
605 return int(subprocess.check_output(["nproc", "--all"]))
607 def parse_netlinks(self, ifaces):
608 """Parse a list of strings, or a single string.
610 Looks if the interfaces exist and configures extra parameters (or
611 properties) -> ie. ['mtu:9000', 'speed:1000', 'op:up']
612 """
613 iface_path = "/sys/class/net/{}"
614 props_dict = {"mtu": "-m {}", "speed": "-s {}", "op": "-o {}"}
615 if type(ifaces) == str:
616 ifaces = [ifaces]
618 d_ifaces = {}
619 for iface in ifaces:
620 iface_props = iface.strip().split()
621 # no ifaces defined; SKIP
622 if len(iface_props) == 0:
623 continue
625 target = iface_props[0]
626 try:
627 matches = match_cidr_to_ifaces(target)
628 except Exception as e:
629 # Log likely unintentional errors and set flag for blocked status,
630 # if appropriate.
631 if isinstance(e, ValueError) and "has host bits set" in e.args[0]:
632 hookenv.log(
633 "Error parsing netlinks: {}".format(e.args[0]),
634 level=hookenv.ERROR,
635 )
636 set_netlinks_error()
637 # Treat target as explicit interface name
638 matches = [target]
640 iface_devs = [
641 target
642 for target in matches
643 if os.path.exists(iface_path.format(target))
644 ]
645 # no ifaces found; SKIP
646 if not iface_devs:
647 continue
649 # parse extra parameters (properties)
650 del iface_props[0]
651 extra_params = ""
652 for prop in iface_props:
653 # wrong format (key:value); SKIP
654 if prop.find(":") < 0:
655 continue
657 # only one ':' expected
658 kv = prop.split(":")
659 if len(kv) == 2 and kv[0].lower() in props_dict:
660 extra_params += " "
661 extra_params += props_dict[kv[0].lower()].format(kv[1])
663 for iface_dev in iface_devs:
664 d_ifaces[iface_dev] = "-i {}{}".format(iface_dev, extra_params)
665 return d_ifaces
668def match_cidr_to_ifaces(cidr):
669 """Use CIDR expression to search for matching network adapters.
671 Returns a list of adapter names.
672 """
673 import netifaces # Avoid import error before this dependency gets installed
675 network = ipaddress.IPv4Network(cidr)
676 matches = []
677 for adapter in netifaces.interfaces():
678 ipv4_addr_structs = netifaces.ifaddresses(adapter).get(netifaces.AF_INET, [])
679 addrs = [
680 ipaddress.IPv4Address(addr_struct["addr"])
681 for addr_struct in ipv4_addr_structs
682 ]
683 if any(addr in network for addr in addrs):
684 matches.append(adapter)
685 return matches
688def has_netlinks_error():
689 """Return True in case of netlinks related errors."""
690 return NETLINKS_ERROR
693def set_netlinks_error():
694 """Set the flag indicating a netlinks related error."""
695 global NETLINKS_ERROR
696 NETLINKS_ERROR = True