Coverage for lib/haproxy.py: 100%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

282 statements  

1import hashlib 

2import multiprocessing 

3import os 

4import re 

5import subprocess 

6import socket 

7 

8import jinja2 

9from distutils.version import LooseVersion 

10 

11from lib import utils 

12 

13 

14HAPROXY_BASE_PATH = '/etc/haproxy' 

15HAPROXY_LOAD_BALANCING_ALGORITHM = 'leastconn' 

16HAPROXY_SAVED_SERVER_STATE_PATH = '/run/haproxy/saved-server-state' 

17HAPROXY_SOCKET_PATH = '/run/haproxy/admin.sock' 

18INDENT = ' ' * 4 

19TLS_CIPHER_SUITES = 'ECDHE+AESGCM:ECDHE+AES256:ECDHE+AES128:!SSLv3:!TLSv1' 

20 

21 

22class HAProxyConf: 

23 def __init__( 

24 self, conf_path=HAPROXY_BASE_PATH, max_connections=0, hard_stop_after='5m', load_balancing_algorithm=None 

25 ): 

26 self._conf_path = conf_path 

27 self.max_connections = int(max_connections) 

28 self.hard_stop_after = hard_stop_after 

29 self.load_balancing_algorithm = HAPROXY_LOAD_BALANCING_ALGORITHM 

30 if load_balancing_algorithm: 

31 self.load_balancing_algorithm = load_balancing_algorithm 

32 self.saved_server_state_path = HAPROXY_SAVED_SERVER_STATE_PATH 

33 self.socket_path = HAPROXY_SOCKET_PATH 

34 

35 @property 

36 def conf_path(self): 

37 return self._conf_path 

38 

39 @property 

40 def conf_file(self): 

41 return os.path.join(self._conf_path, 'haproxy.cfg') 

42 

43 @property 

44 def monitoring_password(self): 

45 try: 

46 with open(self.conf_file, 'r') as f: 

47 m = re.search(r"stats auth\s+(\w+):(\w+)", f.read()) 

48 if m is not None: 

49 return m.group(2) 

50 else: 

51 return None 

52 except FileNotFoundError: 

53 return None 

54 

55 def _generate_stanza_name(self, name, exclude=None): 

56 if exclude is None: 

57 exclude = [] 

58 if len(name) > 32: 

59 # We want to limit the stanza name to 32 characters, but if we 

60 # merely take the first 32 characters of the name, there's a 

61 # chance of collision. We can reduce (but not eliminate) 

62 # this possibility by including a hash fragment of the full 

63 # original name in the result. 

64 name_hash = hashlib.md5(name.encode('UTF-8')).hexdigest() 

65 name = name.replace('.', '-')[0:24] + '-' + name_hash[0:7] 

66 else: 

67 name = name.replace('.', '-') 

68 if name not in exclude: 

69 return name 

70 count = 2 

71 while True: 

72 new_name = '{}-{}'.format(name, count) 

73 count += 1 

74 if new_name not in exclude: 

75 return new_name 

76 

77 def _merge_listen_stanzas(self, config): 

78 new = {} 

79 for site in config.keys(): 

80 site_name = config[site].get('site-name', site) 

81 listen_address = config[site].get('listen-address', '0.0.0.0') 

82 default_port = 80 

83 tls_cert_bundle_path = config[site].get('tls-cert-bundle-path') 

84 if tls_cert_bundle_path: 

85 default_port = 443 

86 if config[site].get('redirect-http-to-https'): 

87 new.setdefault('0.0.0.0:80', {}) 

88 # We use a different flag/config here so it's only enabled 

89 # on the HTTP, and not the HTTPS, stanza. 

90 new['0.0.0.0:80'][site_name] = {'enable-redirect-http-to-https': True} 

91 if 'default' in config[site]: 

92 new['0.0.0.0:80'][site_name]['default'] = config[site]['default'] 

93 

94 port = config[site].get('port', default_port) 

95 name = '{}:{}'.format(listen_address, port) 

96 new.setdefault(name, {}) 

97 new[name][site] = config[site] 

98 new[name][site]['port'] = port 

99 

100 for location, loc_conf in config[site].get('locations', {}).items(): 

101 if 'backend_port' in loc_conf: 

102 port = loc_conf['backend_port'] 

103 name = '{}:{}'.format(listen_address, port) 

104 new.setdefault(name, {}) 

105 count = 2 

106 new_site = site 

107 while new_site in new[name]: 

108 new_site = '{}-{}'.format(site, count) 

109 count += 1 

110 if site not in new[name] or new[name][site]['port'] != port: 

111 new[name][new_site] = config[site] 

112 new[name][new_site]['port'] = port 

113 return new 

114 

115 def render_stanza_listen(self, config): # NOQA: C901 

116 listen_stanza = """ 

117listen {name} 

118{bind_config} 

119{indent}capture request header X-Cache-Request-ID len 60 

120{redirect_config}{backend_config}{default_backend}""" 

121 backend_conf = '{indent}use_backend backend-{backend} if {{ hdr(Host) -i {site_name} }}\n' 

122 redirect_conf = '{indent}redirect scheme https code 301 if {{ hdr(Host) -i {site_name} }} !{{ ssl_fc }}\n' 

123 

124 rendered_output = [] 

125 stanza_names = [] 

126 

127 # For listen stanzas, we need to merge them and use 'use_backend' with 

128 # the 'Host' header to direct to the correct backends. 

129 config = self._merge_listen_stanzas(config) 

130 for address_port in config: 

131 (address, port) = utils.ip_addr_port_split(address_port) 

132 

133 backend_config = [] 

134 default_backend = '' 

135 redirect_config = [] 

136 tls_cert_bundle_paths = [] 

137 redirect_http_to_https = False 

138 for site, site_conf in config[address_port].items(): 

139 site_name = site_conf.get('site-name', site) 

140 default_site = site_conf.get('default', False) 

141 redirect_http_to_https = site_conf.get('enable-redirect-http-to-https', False) 

142 

143 if len(config[address_port].keys()) == 1: 

144 new_site = site 

145 if redirect_http_to_https: 

146 new_site = 'redirect-{}'.format(site) 

147 name = self._generate_stanza_name(new_site, stanza_names) 

148 else: 

149 name = 'combined-{}'.format(port) 

150 stanza_names.append(name) 

151 

152 tls_path = site_conf.get('tls-cert-bundle-path') 

153 if tls_path: 

154 tls_cert_bundle_paths.append(tls_path) 

155 

156 # HTTP -> HTTPS redirect 

157 if redirect_http_to_https: 

158 redirect_config.append(redirect_conf.format(site_name=site_name, indent=INDENT)) 

159 if default_site: 

160 default_backend = "{indent}redirect prefix https://{site_name}\n".format( 

161 site_name=site_name, indent=INDENT 

162 ) 

163 else: 

164 backend_name = self._generate_stanza_name( 

165 site_conf.get('locations', {}).get('backend-name') or site 

166 ) 

167 backend_config.append(backend_conf.format(backend=backend_name, site_name=site_name, indent=INDENT)) 

168 if default_site: 

169 default_backend = "{indent}default_backend backend-{backend}\n".format( 

170 backend=backend_name, indent=INDENT 

171 ) 

172 

173 tls_config = '' 

174 if tls_cert_bundle_paths: 

175 paths = sorted(set(tls_cert_bundle_paths)) 

176 certs = ' '.join(['crt {}'.format(path) for path in paths]) 

177 alpn_protos = 'h2,http/1.1' 

178 tls_config = ' ssl {} alpn {}'.format(certs, alpn_protos) 

179 

180 if len(backend_config) + len(redirect_config) == 1: 

181 if redirect_http_to_https: 

182 redirect_config = [] 

183 default_backend = "{indent}redirect prefix https://{site_name}\n".format( 

184 site_name=site_name, indent=INDENT 

185 ) 

186 else: 

187 backend_config = [] 

188 default_backend = "{indent}default_backend backend-{backend}\n".format(backend=name, indent=INDENT) 

189 

190 bind_config = '{indent}bind {address_port}{tls}'.format( 

191 address_port=address_port, tls=tls_config, indent=INDENT 

192 ) 

193 # Handle 0.0.0.0 and also listen on IPv6 interfaces 

194 if address == '0.0.0.0': 

195 bind_config += '\n{indent}bind :::{port}{tls}'.format(port=port, tls=tls_config, indent=INDENT) 

196 

197 # Redirects are always processed before use_backends so we 

198 # need to convert default redirect sites to a backend. 

199 if len(backend_config) + len(redirect_config) > 1 and default_backend.startswith( 

200 "{indent}redirect prefix".format(indent=INDENT) 

201 ): 

202 backend_name = self._generate_stanza_name("default-redirect-{}".format(name), exclude=stanza_names) 

203 output = "backend {}\n".format(backend_name) + default_backend 

204 default_backend = "{indent}default_backend {backend_name}\n".format( 

205 backend_name=backend_name, indent=INDENT 

206 ) 

207 rendered_output.append(output) 

208 stanza_names.append(backend_name) 

209 

210 output = listen_stanza.format( 

211 name=name, 

212 backend_config=''.join(backend_config), 

213 bind_config=bind_config, 

214 default_backend=default_backend, 

215 redirect_config=''.join(redirect_config), 

216 indent=INDENT, 

217 ) 

218 rendered_output.append(output) 

219 

220 return rendered_output 

221 

222 def render_stanza_backend(self, config): # NOQA: C901 

223 backend_stanza = """ 

224backend backend-{name} 

225{indent}{httpchk} 

226{indent}http-request set-header Host {site_name} 

227{options}{indent}balance {load_balancing_algorithm} 

228{backends} 

229""" 

230 rendered_output = [] 

231 for site, site_conf in config.items(): 

232 backends = [] 

233 

234 for location, loc_conf in site_conf.get('locations', {}).items(): 

235 # No backends, so nothing needed 

236 if not loc_conf.get('backends'): 

237 continue 

238 

239 site_name = loc_conf.get('site-name', site_conf.get('site-name', site)) 

240 

241 tls_config = '' 

242 if loc_conf.get('backend-tls'): 

243 tls_config = ( 

244 ' ssl sni str({site_name}) check-sni {site_name} verify required' 

245 ' ca-file ca-certificates.crt alpn h2,http/1.1'.format(site_name=site_name) 

246 ) 

247 ver = utils.package_version('haproxy') 

248 # With HAProxy 2, we also need check-alpn 

249 if LooseVersion(ver) >= LooseVersion('2'): 

250 tls_config += ' check-alpn h2,http/1.1' 

251 inter_time = loc_conf.get('backend-inter-time', '5s') 

252 fall_count = loc_conf.get('backend-fall-count', 5) 

253 rise_count = loc_conf.get('backend-rise-count', 2) 

254 maxconn = loc_conf.get('backend-maxconn', 200) 

255 method = loc_conf.get('backend-check-method', 'HEAD') 

256 path = loc_conf.get('backend-check-path', '/') 

257 signed_url_hmac_key = loc_conf.get('signed-url-hmac-key') 

258 if signed_url_hmac_key: 

259 expiry_time = utils.never_expires_time() 

260 path = '{}?token={}'.format(path, utils.generate_token(signed_url_hmac_key, path, expiry_time)) 

261 

262 # There may be more than one backend for a site, we need to deal 

263 # with it and ensure our name for the backend stanza is unique. 

264 backend_name = self._generate_stanza_name(site, backends) 

265 backends.append(backend_name) 

266 

267 backend_confs = [] 

268 count = 0 

269 for backend_flags in loc_conf.get('backends'): 

270 flags = backend_flags.split() 

271 backend = flags.pop(0) 

272 

273 count += 1 

274 name = 'server server_{}'.format(count) 

275 

276 backup = '' 

277 for flag in flags: 

278 # https://www.haproxy.com/documentation/hapee/1-8r2/traffic-management/dns-service-discovery/dns-srv-records/ 

279 if flag == 'srv': 

280 name = 'server-template server_{}_ {}'.format(count, flags[flags.index(flag) + 1]) 

281 elif flag == 'backup': 

282 backup = ' backup' 

283 

284 use_resolvers = '' 

285 try: 

286 utils.ip_addr_port_split(backend) 

287 except utils.InvalidAddressPortError: 

288 # http://cbonte.github.io/haproxy-dconv/2.2/configuration.html#init-addr 

289 # "last" - pick the address which appears in the state file 

290 # "libc" - use the libc's internal resolver. 

291 # "none" - start without any valid IP address in a down state 

292 use_resolvers = ' resolvers dns init-addr last,libc,none' 

293 backend_confs.append( 

294 '{indent}{name} {backend}{backup}{use_resolvers} check inter {inter_time} ' 

295 'rise {rise_count} fall {fall_count} maxconn {maxconn}{tls}'.format( 

296 name=name, 

297 backend=backend, 

298 backup=backup, 

299 use_resolvers=use_resolvers, 

300 inter_time=inter_time, 

301 fall_count=fall_count, 

302 rise_count=rise_count, 

303 maxconn=maxconn, 

304 tls=tls_config, 

305 indent=INDENT, 

306 ) 

307 ) 

308 

309 opts = [] 

310 for option in loc_conf.get('backend-options', []): 

311 # retry-on only available from HAProxy 2.1. 

312 if option.split()[0] == 'retry-on' and LooseVersion( 

313 utils.package_version('haproxy') 

314 ) <= LooseVersion('2.1'): 

315 continue 

316 prefix = '' 

317 if option.split()[0] in ['allbackups', 'forceclose', 'forwardfor', 'redispatch']: 

318 prefix = 'option ' 

319 opts.append('{indent}{prefix}{opt}'.format(opt=option, prefix=prefix, indent=INDENT)) 

320 

321 options = '' 

322 if opts: 

323 options = '\n'.join(opts + ['']) 

324 

325 httpchk = ( 

326 r"option httpchk {method} {path} HTTP/1.1\r\n" 

327 r"Host:\ {site_name}\r\n" 

328 r"User-Agent:\ haproxy/httpchk" 

329 ).format(method=method, path=path, site_name=site_name) 

330 

331 output = backend_stanza.format( 

332 name=backend_name, 

333 site=site, 

334 site_name=site_name, 

335 httpchk=httpchk, 

336 load_balancing_algorithm=self.load_balancing_algorithm, 

337 backends='\n'.join(backend_confs), 

338 options=options, 

339 indent=INDENT, 

340 ) 

341 

342 rendered_output.append(output) 

343 

344 return rendered_output 

345 

346 def _calculate_num_procs_threads(self, num_procs, num_threads): 

347 if num_procs and num_threads: 

348 ver = utils.package_version('haproxy') 

349 # With HAProxy 2, nbproc and nbthreads are mutually exclusive. 

350 if LooseVersion(ver) >= LooseVersion('2'): 

351 num_threads = num_procs * num_threads 

352 num_procs = 0 

353 elif not num_procs and not num_threads: 

354 num_threads = multiprocessing.cpu_count() 

355 if not num_procs: 

356 num_procs = 0 

357 if not num_threads: 

358 num_threads = 0 

359 # Assume 64-bit CPU so limit processes and threads to 64. 

360 # https://discourse.haproxy.org/t/architectural-limitation-for-nbproc/5270 

361 num_procs = min(64, num_procs) 

362 num_threads = min(64, num_threads) 

363 return (num_procs, num_threads) 

364 

365 def render(self, config, num_procs=None, num_threads=None, monitoring_password=None, tls_cipher_suites=None): 

366 (num_procs, num_threads) = self._calculate_num_procs_threads(num_procs, num_threads) 

367 

368 listen_stanzas = self.render_stanza_listen(config) 

369 

370 if self.max_connections: 

371 max_connections = self.max_connections 

372 else: 

373 max_connections = num_threads * 2000 

374 global_max_connections = max_connections * len(listen_stanzas) 

375 

376 # Little buffer for non-connection related file descriptors 

377 # such as logging. 

378 fds_buffer = (len(listen_stanzas) * 13) + 1000 

379 maxfds = (global_max_connections * 2) + fds_buffer 

380 init_maxfds = utils.process_rlimits(1, 'NOFILE') 

381 # Calculated max. fds larger than init / PID 1's so let's reduce it. 

382 if init_maxfds != 'unlimited' and maxfds > int(init_maxfds): 

383 global_max_connections = (int(init_maxfds) // 2) - fds_buffer 

384 maxfds = init_maxfds 

385 # Increase max. fds for the HAProxy process, if it needs to. 

386 self.increase_maxfds(self.get_parent_pid(), maxfds) 

387 

388 if not tls_cipher_suites: 

389 tls_cipher_suites = TLS_CIPHER_SUITES 

390 tls_cipher_suites = utils.tls_cipher_suites(tls_cipher_suites) 

391 

392 base = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 

393 env = jinja2.Environment(loader=jinja2.FileSystemLoader(base)) 

394 template = env.get_template('templates/haproxy_cfg.tmpl') 

395 return template.render( 

396 { 

397 'backend': self.render_stanza_backend(config), 

398 'dns_servers': utils.dns_servers(), 

399 'global_max_connections': global_max_connections, 

400 'hard_stop_after': self.hard_stop_after, 

401 'listen': listen_stanzas, 

402 'max_connections': max_connections, 

403 'monitoring_password': monitoring_password or self.monitoring_password, 

404 'num_procs': num_procs, 

405 'num_threads': num_threads, 

406 'saved_server_state_path': self.saved_server_state_path, 

407 'socket_path': self.socket_path, 

408 'tls_cipher_suites': tls_cipher_suites, 

409 } 

410 ) 

411 

412 def write(self, content): 

413 # Check if contents changed 

414 try: 

415 with open(self.conf_file, 'r', encoding='utf-8') as f: 

416 current = f.read() 

417 except FileNotFoundError: 

418 current = '' 

419 if content == current: 

420 return False 

421 with open(self.conf_file, 'w', encoding='utf-8') as f: 

422 f.write(content) 

423 return True 

424 

425 def get_parent_pid(self, pidfile='/run/haproxy.pid'): 

426 if not os.path.exists(pidfile): 

427 # No HAProxy process running, so return PID of init. 

428 return 1 

429 with open(pidfile) as f: 

430 return int(f.readline().strip()) 

431 

432 # HAProxy 2.x does this, but Bionic ships with HAProxy 1.8 so we need 

433 # to still do this. 

434 def increase_maxfds(self, haproxy_pid, maxfds): 

435 haproxy_maxfds = utils.process_rlimits(haproxy_pid, 'NOFILE') 

436 

437 if haproxy_maxfds and haproxy_maxfds != 'unlimited' and int(maxfds) > int(haproxy_maxfds): 

438 cmd = ['prlimit', '--pid', str(haproxy_pid), '--nofile={}'.format(str(maxfds))] 

439 subprocess.call(cmd, stdout=subprocess.DEVNULL) 

440 return True 

441 

442 return False 

443 

444 def save_server_state(self): 

445 server_state = b"" 

446 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: 

447 s.connect(self.socket_path) 

448 s.settimeout(5.0) 

449 s.sendall(b"show servers state\n") 

450 while True: 

451 data = s.recv(1024) 

452 if not data: 

453 break 

454 server_state += data 

455 

456 new_state = "{}.new".format(self.saved_server_state_path) 

457 with open(new_state, "wb") as f: 

458 f.write(server_state) 

459 os.rename(new_state, self.saved_server_state_path)