Coverage for lib/nginx.py: 64%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

181 statements  

1import hashlib 

2import os 

3import re 

4from copy import deepcopy 

5 

6import jinja2 

7 

8from lib import utils 

9 

10 

11INDENT = ' ' * 4 

12METRICS_LISTEN = 'localhost' 

13METRICS_PORT = 9145 

14METRICS_SITE = 'nginx_metrics' 

15NGINX_BASE_PATH = '/etc/nginx' 

16# Subset of http://nginx.org/en/docs/http/ngx_http_proxy_module.html 

17PROXY_CACHE_DEFAULTS = { 

18 'background-update': 'on', 

19 'lock': 'on', 

20 'min-uses': 1, 

21 'revalidate': 'on', 

22 'use-stale': 'error timeout updating http_500 http_502 http_503 http_504', 

23 'valid': ['200 1d'], 

24} 

25 

26 

27class NginxConf: 

28 def __init__(self, conf_path=None, unit='content-cache', enable_cache_bg_update=True, enable_cache_lock=True): 

29 if not conf_path: 29 ↛ 30line 29 didn't jump to line 30, because the condition on line 29 was never true

30 conf_path = NGINX_BASE_PATH 

31 self.unit = unit 

32 

33 self._base_path = conf_path 

34 self._conf_path = os.path.join(self.base_path, 'conf.d') 

35 self._enable_cache_bg_update = enable_cache_bg_update 

36 self._enable_cache_lock = enable_cache_lock 

37 self._sites_path = os.path.join(self.base_path, 'sites-available') 

38 

39 script_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 

40 self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(script_dir)) 

41 

42 # Expose base_path as a property to allow mocking in indirect calls to 

43 # this class. 

44 @property 

45 def base_path(self): 

46 return self._base_path 

47 

48 # Expose conf_path as a property to allow mocking in indirect calls to 

49 # this class. 

50 @property 

51 def conf_path(self): 

52 return self._conf_path 

53 

54 # Expose sites_path as a property to allow mocking in indirect calls to 

55 # this class. 

56 @property 

57 def sites_path(self): 

58 return self._sites_path 

59 

60 # Expose sites_path as a property to allow mocking in indirect calls to 

61 # this class. 

62 @property 

63 def proxy_cache_configs(self): 

64 return PROXY_CACHE_DEFAULTS 

65 

66 def write_site(self, site, new): 

67 fname = os.path.join(self.sites_path, '{}.conf'.format(site)) 

68 # Check if contents changed 

69 try: 

70 with open(fname, 'r', encoding='utf-8') as f: 

71 current = f.read() 

72 except FileNotFoundError: 

73 current = '' 

74 if new == current: 74 ↛ 75line 74 didn't jump to line 75, because the condition on line 74 was never true

75 return False 

76 with open(fname, 'w', encoding='utf-8') as f: 

77 f.write(new) 

78 return True 

79 

80 def sync_sites(self, sites): 

81 changed = False 

82 for fname in os.listdir(self.sites_path): 

83 site = fname.replace('.conf', '') 

84 available = os.path.join(self.sites_path, fname) 

85 enabled = os.path.join(os.path.dirname(self.sites_path), 'sites-enabled', fname) 

86 if site not in sites: 86 ↛ 87line 86 didn't jump to line 87, because the condition on line 86 was never true

87 changed = True 

88 try: 

89 os.remove(available) 

90 os.remove(enabled) 

91 except FileNotFoundError: 

92 pass 

93 elif not os.path.exists(enabled): 

94 changed = True 

95 os.symlink(available, enabled) 

96 

97 return changed 

98 

99 def _generate_keys_zone(self, name): 

100 return '{}-cache'.format(hashlib.md5(name.encode('UTF-8')).hexdigest()[0:12]) 

101 

102 def _process_locations(self, locations): # NOQA: C901 

103 conf = {} 

104 for location, loc_conf in locations.items(): 

105 conf[location] = deepcopy(loc_conf) 

106 lc = conf[location] 

107 backend_port = lc.get('backend_port') 

108 if backend_port: 108 ↛ 145line 108 didn't jump to line 145, because the condition on line 108 was never false

109 backend_path = lc.get('backend-path') 

110 lc['backend'] = utils.generate_uri('localhost', backend_port, backend_path) 

111 for k, v in self.proxy_cache_configs.items(): 

112 cache_key = 'cache-{}'.format(k) 

113 if cache_key == 'cache-valid': 

114 # Skip and set the default later. 

115 continue 

116 lc.setdefault(cache_key, v) 

117 

118 if not self._enable_cache_bg_update: 118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true

119 lc['cache-background-update'] = 'off' 

120 lc['cache-use-stale'] = lc['cache-use-stale'].replace('updating ', '') 

121 

122 if not self._enable_cache_lock: 122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true

123 lc['cache-lock'] = 'off' 

124 

125 cache_val = self.proxy_cache_configs['valid'] 

126 # Backwards compatibility 

127 if 'cache-validity' in lc: 127 ↛ 128line 127 didn't jump to line 128, because the condition on line 127 was never true

128 cache_val = lc['cache-validity'] 

129 lc.pop('cache-validity') 

130 elif 'cache-valid' in lc: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true

131 cache_val = lc['cache-valid'] 

132 

133 # No such thing as proxy_cache_maxconn, this is more used by 

134 # HAProxy so remove/ignore here. 

135 if 'cache-maxconn' in lc: 135 ↛ 136line 135 didn't jump to line 136, because the condition on line 135 was never true

136 lc.pop('cache-maxconn') 

137 

138 lc['cache-valid'] = [] 

139 # Support multiple cache-validities per LP:1873116. 

140 if isinstance(cache_val, str): 140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true

141 lc['cache-valid'].append(cache_val) 

142 else: 

143 lc['cache-valid'] += cache_val 

144 

145 lc['force_ranges'] = 'on' 

146 extra_config = lc.get('extra-config', []) 

147 for ext in extra_config: 147 ↛ 148line 147 didn't jump to line 148, because the loop on line 147 never started

148 if ext.startswith('proxy_force_ranges'): 

149 lc['force_ranges'] = ext.split()[1] 

150 extra_config.remove(ext) 

151 

152 return conf 

153 

154 def render(self, conf): 

155 data = { 

156 'address': conf['listen_address'], 

157 'cache_inactive_time': conf['cache_inactive_time'], 

158 'cache_max_size': conf['cache_max_size'], 

159 'cache_path': conf['cache_path'], 

160 'enable_prometheus_metrics': conf['enable_prometheus_metrics'], 

161 'juju_unit': self.unit, 

162 'keys_zone': self._generate_keys_zone(conf['site']), 

163 'locations': self._process_locations(conf['locations']), 

164 'port': conf['listen_port'], 

165 'reuseport': conf['reuseport'], 

166 'site': conf['site'], 

167 'site_name': conf['site_name'], 

168 } 

169 template = self.jinja_env.get_template('templates/nginx_cfg.tmpl') 

170 return template.render(data) 

171 

172 def _remove_metrics_site(self, available, enabled): 

173 """Remove the configuration exposing metrics. 

174 

175 :param str available: Path of the "available" site exposing the metrics 

176 :param str enabled: Path of the "enabled" symlink to the "available" configuration 

177 :returns: True if any change was made, False otherwise 

178 :rtype: bool 

179 """ 

180 changed = False 

181 try: 

182 os.remove(available) 

183 changed = True 

184 except FileNotFoundError: 

185 pass 

186 try: 

187 os.remove(enabled) 

188 changed = True 

189 except FileNotFoundError: 

190 pass 

191 

192 return changed 

193 

194 def toggle_metrics_site(self, enable_prometheus_metrics, listen_address=None): 

195 """Create/delete the metrics site configuration and links. 

196 

197 :param bool enable_prometheus_metrics: True if metrics are exposed to prometheus 

198 :returns: True if any change was made, False otherwise 

199 :rtype: bool 

200 """ 

201 changed = False 

202 metrics_site_conf = '{0}.conf'.format(METRICS_SITE) 

203 available = os.path.join(self.sites_path, metrics_site_conf) 

204 enabled = os.path.join(self.base_path, 'sites-enabled', metrics_site_conf) 

205 # If no cache metrics, remove the site 

206 if not enable_prometheus_metrics: 206 ↛ 209line 206 didn't jump to line 209, because the condition on line 206 was never false

207 return self._remove_metrics_site(available, enabled) 

208 

209 if listen_address is None: 

210 listen_address = METRICS_LISTEN 

211 

212 template = self.jinja_env.get_template('templates/nginx_metrics_cfg.tmpl') 

213 content = template.render({'nginx_conf_path': self.conf_path, 'address': listen_address, 'port': METRICS_PORT}) 

214 # Check if contents changed 

215 try: 

216 with open(available, 'r', encoding='utf-8') as f: 

217 current = f.read() 

218 except FileNotFoundError: 

219 current = '' 

220 if content != current: 

221 with open(available, 'w', encoding='utf-8') as f: 

222 f.write(content) 

223 changed = True 

224 os.listdir(self.sites_path) 

225 if not os.path.exists(enabled): 

226 os.symlink(available, enabled) 

227 changed = True 

228 if os.path.realpath(available) != os.path.realpath(enabled): 

229 os.remove(enabled) 

230 os.symlink(available, enabled) 

231 changed = True 

232 

233 return changed 

234 

235 def get_workers(self): 

236 nginx_conf_file = os.path.join(self._base_path, 'nginx.conf') 

237 with open(nginx_conf_file, 'r', encoding='utf-8') as f: 

238 content = f.read().split('\n') 

239 res = {'worker_processes': None, 'worker_connections': None} 

240 regex = re.compile(r'^(?:\s+)?(worker_processes|worker_connections)(?:\s+)(\S+).*;') 

241 for line in content: 

242 m = regex.match(line) 

243 if m: 

244 res[m.group(1)] = m.group(2) 

245 return res['worker_connections'], res['worker_processes'] 

246 

247 def set_workers(self, connections, processes): 

248 nginx_conf_file = os.path.join(self._base_path, 'nginx.conf') 

249 

250 val = {'worker_processes': processes, 'worker_connections': connections} 

251 if processes == 0: 251 ↛ 254line 251 didn't jump to line 254, because the condition on line 251 was never false

252 val['worker_processes'] = 'auto' 

253 

254 with open(nginx_conf_file, 'r', encoding='utf-8') as f: 

255 content = f.read().split('\n') 

256 

257 new = [] 

258 regex = re.compile(r'^(\s*(worker_processes|worker_connections))(\s+).*;') 

259 for line in content: 

260 m = regex.match(line) 

261 if m: 

262 new.append('{}{}{};'.format(m.group(1), m.group(3), val[m.group(2)])) 

263 else: 

264 new.append(line) 

265 

266 # Check if contents changed 

267 if new == content: 267 ↛ 269line 267 didn't jump to line 269, because the condition on line 267 was never false

268 return False 

269 with open(nginx_conf_file, 'w', encoding='utf-8') as f: 

270 f.write('\n'.join(new)) 

271 return True