Coverage for lib/utils.py: 56%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import datetime
2import hashlib
3import hmac
4import mmap
5import os
6import re
7import shutil
8import subprocess
10import psutil
13BASE_CACHE_PORT = 6080
14BASE_BACKEND_PORT = 8080
15BACKEND_PORT_LIMIT = 61000 # sysctl net.ipv4.ip_local_port_range
18class InvalidPortError(Exception):
19 pass
22class InvalidAddressPortError(Exception):
23 pass
26class InvalidTLSCiphersError(Exception):
27 pass
30def next_port_pair(
31 cache_port,
32 backend_port,
33 base_cache_port=BASE_CACHE_PORT,
34 base_backend_port=BASE_BACKEND_PORT,
35 backend_port_limit=BACKEND_PORT_LIMIT,
36 blacklist_ports=None,
37):
38 if blacklist_ports is None: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true
39 blacklist_ports = []
41 if cache_port == 0:
42 cache_port = base_cache_port
43 else:
44 cache_port += 1
45 while cache_port in blacklist_ports:
46 cache_port += 1
48 if backend_port == 0:
49 backend_port = base_backend_port
50 else:
51 backend_port += 1
52 while backend_port in blacklist_ports:
53 backend_port += 1
55 if cache_port < base_cache_port or cache_port >= base_backend_port: 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true
56 raise InvalidPortError('Dynamically allocated cache_port out of range')
58 port_limit = base_backend_port + (base_backend_port - base_cache_port)
59 if port_limit >= backend_port_limit: 59 ↛ 60line 59 didn't jump to line 60, because the condition on line 59 was never true
60 port_limit = backend_port_limit
62 if backend_port < base_backend_port or backend_port >= port_limit: 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true
63 raise InvalidPortError('Dynamically allocated backend_port out of range')
65 return (cache_port, backend_port)
68def _nagios_check_name_strip(name):
69 return name.replace('.', '_').replace('-', '_').replace('/', '').replace('__', '_').strip('_')
72def generate_nagios_check_name(name, prefix='', suffix=''):
73 check_name = name
74 if prefix: 74 ↛ 76line 74 didn't jump to line 76, because the condition on line 74 was never false
75 check_name = '{}_{}'.format(prefix, check_name)
76 if suffix: 76 ↛ 78line 76 didn't jump to line 78, because the condition on line 76 was never false
77 check_name = '{}_{}'.format(check_name, suffix)
78 return _nagios_check_name_strip(check_name)
81def generate_token(signing_secret, url_path, expiry_time):
82 expiration = int(expiry_time.timestamp())
83 string_to_sign = "{0}{1}".format(url_path, expiration)
84 digest = hmac.new(signing_secret.encode(), string_to_sign.encode(), hashlib.sha1)
85 return "{0}_{1}".format(expiration, digest.hexdigest())
88def never_expires_time():
89 dt = datetime.date(datetime.datetime.now().year, 1, 1)
90 tm = datetime.time(00, 00)
91 expiry_time = datetime.datetime.combine(dt, tm) + datetime.timedelta(days=3653)
92 return expiry_time
95def generate_uri(host, port=80, path=None, scheme='http'):
96 if not path: 96 ↛ 99line 96 didn't jump to line 99, because the condition on line 96 was never false
97 path = ''
98 # XXX: Fix to handle if host is an IPv6 literal
99 if path and not path.startswith('/'): 99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true
100 path = '/{}'.format(path)
101 uri = '{scheme}://{host}:{port}{path}'.format(scheme=scheme, host=host, port=port, path=path)
102 return uri
105def cache_max_size(path, percent=75):
106 total = shutil.disk_usage(path)[0]
107 percent = percent / 100
108 gbytes = 1024 * 1024 * 1024
109 return '{}g'.format(max(1, int((total * percent) / gbytes)))
112def ip_addr_port_split(addr_port):
113 addr = None
115 # IPv4
116 regex = re.compile('((\\d{1,3}).(\\d{1,3}).(\\d{1,3}).(\\d{1,3})):(\\d{1,5})')
117 m = regex.match(addr_port)
118 if m:
119 addr, port = m.group(1, 6)
120 for octet in m.group(2, 3, 4, 5):
121 if int(octet) > 255: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true
122 addr = None
123 break
125 # IPv6
126 else:
127 regex = re.compile('\\[([:a-fA-F0-9]+)\\]:(\\d{1,5})')
128 m = regex.match(addr_port)
129 if m: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true
130 addr, port = m.group(1, 2)
131 else:
132 port = addr_port.split(':')[-1]
134 port = int(port)
135 if port > 65535: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true
136 port = None
138 if addr is None or port is None:
139 raise InvalidAddressPortError('Unable to split IP address and port from "{}"'.format(addr_port))
141 return (addr, port)
144def tls_cipher_suites(tls_cipher_suites):
145 cmd = ['openssl', 'ciphers', '--', tls_cipher_suites]
146 try:
147 subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
148 except Exception:
149 raise InvalidTLSCiphersError('Unable to parse provided OpenSSL cipher string "{}"'.format(tls_cipher_suites))
150 return tls_cipher_suites
153def logrotate(path, retention=30, dateext=True):
154 if not os.path.exists(path):
155 return None
157 with open(path, 'r', encoding='utf-8') as f:
158 config = f.read().split('\n')
160 new = []
161 regex = re.compile('^(\\s+)(rotate|dateext)')
162 for line in config:
163 m = regex.match(line)
164 if m:
165 if m.group(2) == 'dateext':
166 continue
167 if dateext:
168 new.append('{}dateext'.format(m.group(1)))
169 new.append('{}rotate {}'.format(m.group(1), retention))
170 else:
171 new.append(line)
173 return '\n'.join(new)
176LIMITS_MATCH = {
177 'NOFILE': 'Max open files',
178}
181def process_rlimits(pid, res, limits_file=None):
182 if not limits_file: 182 ↛ 185line 182 didn't jump to line 185, because the condition on line 182 was never false
183 limits_file = os.path.join('/proc', str(pid), 'limits')
185 if not os.path.exists(limits_file): 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true
186 return None
188 with open(limits_file, 'r', encoding='utf-8') as f:
189 limits = f.read()
191 if res not in LIMITS_MATCH: 191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true
192 return None
194 r = re.compile(r'^{}\s+\S+\s+(\S+)'.format(LIMITS_MATCH[res]))
195 for line in limits.split('\n'): 195 ↛ 200line 195 didn't jump to line 200, because the loop on line 195 didn't complete
196 m = r.match(line)
197 if m:
198 return m.group(1)
200 return None
203def dns_servers(resolvconf_file='/etc/resolv.conf'):
204 servers = []
205 with open(resolvconf_file, 'r', encoding='utf-8') as f:
206 resolvconf = f.read()
207 for line in resolvconf.split('\n'):
208 t = line.split()
209 if not t:
210 continue
211 if t[0] == 'nameserver':
212 servers.append(t[1])
213 return servers
216def package_version(package):
217 cmd = ['dpkg-query', '--show', r'--showformat=${Version}\n', package]
218 try:
219 version = subprocess.check_output(cmd, universal_newlines=True).strip()
220 except subprocess.CalledProcessError:
221 return None
222 if not version:
223 return None
224 return version
227_SYSCTL_NET_IPV4_CONGESTION_CONTROL = '/proc/sys/net/ipv4/tcp_congestion_control'
230def select_tcp_congestion_control(preferred_tcp_cc, tcp_avail_path=_SYSCTL_NET_IPV4_CONGESTION_CONTROL):
231 if not os.path.exists(tcp_avail_path):
232 return None
234 for pref_cc in preferred_tcp_cc:
235 # We need to try set the TCP congestion control algorithm and
236 # see if it's successfully set.
237 cmd = ['sysctl', 'net.ipv4.tcp_congestion_control={}'.format(pref_cc)]
238 subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
240 # Now check to see if it's set in '/proc/sys/net/ipv4/tcp_congestion_control'
241 # and use if so.
242 with open(tcp_avail_path) as f:
243 ccs = f.read()
244 if pref_cc in ccs.split():
245 return pref_cc
247 return None
250_SYSCTL_NET_IPV4_TCP_MEM = '/proc/sys/net/ipv4/tcp_mem'
253def tune_tcp_mem(multiplier=1.5, tcp_mem_path=_SYSCTL_NET_IPV4_TCP_MEM, mmap_pagesize=mmap.PAGESIZE):
255 # For LXC/LXD containers, we can't tune tcp_mem.
256 if not os.path.exists(tcp_mem_path):
257 return None
259 svmem = psutil.virtual_memory()
260 total_mem = svmem.total
262 # Try to calculate the system defaults for tcp_mem based on code
263 # from the kernel:
264 # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv4/tcp.c?id=3aa7857fe1d7ac7f600f5b7e1530396fb06822bf#n4487
265 # We can't use the available memory unlike what the kernel uses as
266 # that differs for an already running system vs. on initial system
267 # boot. Let's go with 98.8%
268 total_pages = (total_mem * 0.988) / mmap_pagesize
269 limit = total_pages / 16
270 mem_min = limit / 4 * 3
271 mem_pressure = limit
272 mem_max = mem_min * 2
274 return "{} {} {}".format(int(mem_min * multiplier), int(mem_pressure * multiplier), int(mem_max * multiplier))