Coverage for lib/utils.py: 56%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

175 statements  

1import datetime 

2import hashlib 

3import hmac 

4import mmap 

5import os 

6import re 

7import shutil 

8import subprocess 

9 

10import psutil 

11 

12 

13BASE_CACHE_PORT = 6080 

14BASE_BACKEND_PORT = 8080 

15BACKEND_PORT_LIMIT = 61000 # sysctl net.ipv4.ip_local_port_range 

16 

17 

18class InvalidPortError(Exception): 

19 pass 

20 

21 

22class InvalidAddressPortError(Exception): 

23 pass 

24 

25 

26class InvalidTLSCiphersError(Exception): 

27 pass 

28 

29 

30def next_port_pair( 

31 cache_port, 

32 backend_port, 

33 base_cache_port=BASE_CACHE_PORT, 

34 base_backend_port=BASE_BACKEND_PORT, 

35 backend_port_limit=BACKEND_PORT_LIMIT, 

36 blacklist_ports=None, 

37): 

38 if blacklist_ports is None: 38 ↛ 39line 38 didn't jump to line 39, because the condition on line 38 was never true

39 blacklist_ports = [] 

40 

41 if cache_port == 0: 

42 cache_port = base_cache_port 

43 else: 

44 cache_port += 1 

45 while cache_port in blacklist_ports: 

46 cache_port += 1 

47 

48 if backend_port == 0: 

49 backend_port = base_backend_port 

50 else: 

51 backend_port += 1 

52 while backend_port in blacklist_ports: 

53 backend_port += 1 

54 

55 if cache_port < base_cache_port or cache_port >= base_backend_port: 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true

56 raise InvalidPortError('Dynamically allocated cache_port out of range') 

57 

58 port_limit = base_backend_port + (base_backend_port - base_cache_port) 

59 if port_limit >= backend_port_limit: 59 ↛ 60line 59 didn't jump to line 60, because the condition on line 59 was never true

60 port_limit = backend_port_limit 

61 

62 if backend_port < base_backend_port or backend_port >= port_limit: 62 ↛ 63line 62 didn't jump to line 63, because the condition on line 62 was never true

63 raise InvalidPortError('Dynamically allocated backend_port out of range') 

64 

65 return (cache_port, backend_port) 

66 

67 

68def _nagios_check_name_strip(name): 

69 return name.replace('.', '_').replace('-', '_').replace('/', '').replace('__', '_').strip('_') 

70 

71 

72def generate_nagios_check_name(name, prefix='', suffix=''): 

73 check_name = name 

74 if prefix: 74 ↛ 76line 74 didn't jump to line 76, because the condition on line 74 was never false

75 check_name = '{}_{}'.format(prefix, check_name) 

76 if suffix: 76 ↛ 78line 76 didn't jump to line 78, because the condition on line 76 was never false

77 check_name = '{}_{}'.format(check_name, suffix) 

78 return _nagios_check_name_strip(check_name) 

79 

80 

81def generate_token(signing_secret, url_path, expiry_time): 

82 expiration = int(expiry_time.timestamp()) 

83 string_to_sign = "{0}{1}".format(url_path, expiration) 

84 digest = hmac.new(signing_secret.encode(), string_to_sign.encode(), hashlib.sha1) 

85 return "{0}_{1}".format(expiration, digest.hexdigest()) 

86 

87 

88def never_expires_time(): 

89 dt = datetime.date(datetime.datetime.now().year, 1, 1) 

90 tm = datetime.time(00, 00) 

91 expiry_time = datetime.datetime.combine(dt, tm) + datetime.timedelta(days=3653) 

92 return expiry_time 

93 

94 

95def generate_uri(host, port=80, path=None, scheme='http'): 

96 if not path: 96 ↛ 99line 96 didn't jump to line 99, because the condition on line 96 was never false

97 path = '' 

98 # XXX: Fix to handle if host is an IPv6 literal 

99 if path and not path.startswith('/'): 99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true

100 path = '/{}'.format(path) 

101 uri = '{scheme}://{host}:{port}{path}'.format(scheme=scheme, host=host, port=port, path=path) 

102 return uri 

103 

104 

105def cache_max_size(path, percent=75): 

106 total = shutil.disk_usage(path)[0] 

107 percent = percent / 100 

108 gbytes = 1024 * 1024 * 1024 

109 return '{}g'.format(max(1, int((total * percent) / gbytes))) 

110 

111 

112def ip_addr_port_split(addr_port): 

113 addr = None 

114 

115 # IPv4 

116 regex = re.compile('((\\d{1,3}).(\\d{1,3}).(\\d{1,3}).(\\d{1,3})):(\\d{1,5})') 

117 m = regex.match(addr_port) 

118 if m: 

119 addr, port = m.group(1, 6) 

120 for octet in m.group(2, 3, 4, 5): 

121 if int(octet) > 255: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true

122 addr = None 

123 break 

124 

125 # IPv6 

126 else: 

127 regex = re.compile('\\[([:a-fA-F0-9]+)\\]:(\\d{1,5})') 

128 m = regex.match(addr_port) 

129 if m: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true

130 addr, port = m.group(1, 2) 

131 else: 

132 port = addr_port.split(':')[-1] 

133 

134 port = int(port) 

135 if port > 65535: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true

136 port = None 

137 

138 if addr is None or port is None: 

139 raise InvalidAddressPortError('Unable to split IP address and port from "{}"'.format(addr_port)) 

140 

141 return (addr, port) 

142 

143 

144def tls_cipher_suites(tls_cipher_suites): 

145 cmd = ['openssl', 'ciphers', '--', tls_cipher_suites] 

146 try: 

147 subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 

148 except Exception: 

149 raise InvalidTLSCiphersError('Unable to parse provided OpenSSL cipher string "{}"'.format(tls_cipher_suites)) 

150 return tls_cipher_suites 

151 

152 

153def logrotate(path, retention=30, dateext=True): 

154 if not os.path.exists(path): 

155 return None 

156 

157 with open(path, 'r', encoding='utf-8') as f: 

158 config = f.read().split('\n') 

159 

160 new = [] 

161 regex = re.compile('^(\\s+)(rotate|dateext)') 

162 for line in config: 

163 m = regex.match(line) 

164 if m: 

165 if m.group(2) == 'dateext': 

166 continue 

167 if dateext: 

168 new.append('{}dateext'.format(m.group(1))) 

169 new.append('{}rotate {}'.format(m.group(1), retention)) 

170 else: 

171 new.append(line) 

172 

173 return '\n'.join(new) 

174 

175 

176LIMITS_MATCH = { 

177 'NOFILE': 'Max open files', 

178} 

179 

180 

181def process_rlimits(pid, res, limits_file=None): 

182 if not limits_file: 182 ↛ 185line 182 didn't jump to line 185, because the condition on line 182 was never false

183 limits_file = os.path.join('/proc', str(pid), 'limits') 

184 

185 if not os.path.exists(limits_file): 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true

186 return None 

187 

188 with open(limits_file, 'r', encoding='utf-8') as f: 

189 limits = f.read() 

190 

191 if res not in LIMITS_MATCH: 191 ↛ 192line 191 didn't jump to line 192, because the condition on line 191 was never true

192 return None 

193 

194 r = re.compile(r'^{}\s+\S+\s+(\S+)'.format(LIMITS_MATCH[res])) 

195 for line in limits.split('\n'): 195 ↛ 200line 195 didn't jump to line 200, because the loop on line 195 didn't complete

196 m = r.match(line) 

197 if m: 

198 return m.group(1) 

199 

200 return None 

201 

202 

203def dns_servers(resolvconf_file='/etc/resolv.conf'): 

204 servers = [] 

205 with open(resolvconf_file, 'r', encoding='utf-8') as f: 

206 resolvconf = f.read() 

207 for line in resolvconf.split('\n'): 

208 t = line.split() 

209 if not t: 

210 continue 

211 if t[0] == 'nameserver': 

212 servers.append(t[1]) 

213 return servers 

214 

215 

216def package_version(package): 

217 cmd = ['dpkg-query', '--show', r'--showformat=${Version}\n', package] 

218 try: 

219 version = subprocess.check_output(cmd, universal_newlines=True).strip() 

220 except subprocess.CalledProcessError: 

221 return None 

222 if not version: 

223 return None 

224 return version 

225 

226 

227_SYSCTL_NET_IPV4_CONGESTION_CONTROL = '/proc/sys/net/ipv4/tcp_congestion_control' 

228 

229 

230def select_tcp_congestion_control(preferred_tcp_cc, tcp_avail_path=_SYSCTL_NET_IPV4_CONGESTION_CONTROL): 

231 if not os.path.exists(tcp_avail_path): 

232 return None 

233 

234 for pref_cc in preferred_tcp_cc: 

235 # We need to try set the TCP congestion control algorithm and 

236 # see if it's successfully set. 

237 cmd = ['sysctl', 'net.ipv4.tcp_congestion_control={}'.format(pref_cc)] 

238 subprocess.check_call(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 

239 

240 # Now check to see if it's set in '/proc/sys/net/ipv4/tcp_congestion_control' 

241 # and use if so. 

242 with open(tcp_avail_path) as f: 

243 ccs = f.read() 

244 if pref_cc in ccs.split(): 

245 return pref_cc 

246 

247 return None 

248 

249 

250_SYSCTL_NET_IPV4_TCP_MEM = '/proc/sys/net/ipv4/tcp_mem' 

251 

252 

253def tune_tcp_mem(multiplier=1.5, tcp_mem_path=_SYSCTL_NET_IPV4_TCP_MEM, mmap_pagesize=mmap.PAGESIZE): 

254 

255 # For LXC/LXD containers, we can't tune tcp_mem. 

256 if not os.path.exists(tcp_mem_path): 

257 return None 

258 

259 svmem = psutil.virtual_memory() 

260 total_mem = svmem.total 

261 

262 # Try to calculate the system defaults for tcp_mem based on code 

263 # from the kernel: 

264 # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/ipv4/tcp.c?id=3aa7857fe1d7ac7f600f5b7e1530396fb06822bf#n4487 

265 # We can't use the available memory unlike what the kernel uses as 

266 # that differs for an already running system vs. on initial system 

267 # boot. Let's go with 98.8% 

268 total_pages = (total_mem * 0.988) / mmap_pagesize 

269 limit = total_pages / 16 

270 mem_min = limit / 4 * 3 

271 mem_pressure = limit 

272 mem_max = mem_min * 2 

273 

274 return "{} {} {}".format(int(mem_min * multiplier), int(mem_pressure * multiplier), int(mem_max * multiplier))