Coverage for r11k/puppetfile.py: 77%
127 statements
« prev ^ index » next coverage.py v7.2.1, created at 2023-03-13 23:53 +0100
1"""
2Definition and operations on our puppetfile format.
4This file includes:
5- The specification of our puppetfile.yaml format
6 - The thing which publishes our environments
7- the dependency resolver
9## Puppetfile.yaml
11A top level key `modules`, containing a list where each element is a dictionary
12containing
14:param name: Either the forge module name, or the target name for a git repo
15:param [version]: Which version to use.
16:param [git]: Git repo to clone to retrieve this object. Mutually exclusive with `http`
17:param [http]: URL where a tarball can be found. Mutually exclusive with `git`
19If neither a git nor http key is given then it's fetched from the
20[Puppet Forge][forge]
22#### `version`
23If it's a Forge module then the version is looked up in the Forge. If it's a Git
24repo then the version indicates a given ref (tag, branch, commit, ...). For HTTP
25the version field is ignored.
27An absent version field means latest compatible.
29#### `http`
30Is currently not implemented.
32### Sample Puppetfile
34```yaml
35modules:
36 - name: stdlib
37 version: 8.2.0
38 - name: extlib
39 - name: mymodule
40 git: https://git.example.com/a-different-repo-name.git
41```
43--------------------------------------------------
45[forge]: https://forge.puppet.com
46"""
48import logging
49import os.path
51from dataclasses import dataclass, field
52from functools import reduce
53from typing import (
54 Any,
55 Optional,
56)
58import yaml
60import r11k.config
61from r11k import util
62from r11k.interval import Interval
63from r11k import interval
64from r11k.puppetmodule import (
65 PuppetModule,
66 ForgePuppetModule,
67 parse_module_dict,
68)
71logger = logging.getLogger(__name__)
74@dataclass
75class Hiera:
76 """
77 Contents of a hiera.yaml.
79 https://puppet.com/docs/puppet/7/hiera_config_yaml_5.html
80 """
82 version: int = 5
83 hierarchy: list[dict[str, str]] = field(default_factory=list)
84 defaults: Optional[dict[str, str]] = None
85 default_hierarchy: Optional[list[dict[str, str]]] = None
88@dataclass
89class PuppetFile:
90 """Complete contents of a puppetfile.yaml."""
92 # TODO do I really want a dict here? Would a set be better?
93 modules: dict[str, PuppetModule] = field(default_factory=dict)
94 """
95 Module declarations from puppet file. Mapping from module name to
96 module.
97 """
98 environment_name: Optional[str] = None
99 """Name of the environment, used when publishing"""
100 data: dict[str, dict[str, Any]] = field(default_factory=dict)
101 """
102 Hiera data which will be published as part of the environment.
103 Each top level key is a path (with slashes for delimiters), but
104 the .yaml suffix omitted. The dictionaries under that will be
105 written to the output file.
106 """
107 hiera: Optional[Hiera] = None
108 """hiera.yaml for this environment."""
109 config: r11k.config.Config = field(default_factory=lambda: r11k.config.config)
110 """r11k configuration"""
112 def include(self, parent: 'PuppetFile') -> None:
113 """
114 Include a parent to this PuppetFile.
116 ### Merging Strategies
117 #### modules
118 Each module from the parent is included, when both specify the
119 same plugin, the version specified by us gets chosen.
120 #### hiera
121 We keep ours if we have one, otherwise we take it from the
122 parent.
123 #### data
124 The set of files from both instances will be merged,
125 defaulting to our version. The keys inside a file aren't
126 merged.
127 """
128 self.modules = parent.modules | self.modules
130 if not self.hiera: 130 ↛ 133line 130 didn't jump to line 133, because the condition on line 130 was never false
131 self.hiera = parent.hiera
133 self.data = parent.data | self.data
135 def serialize(self) -> dict[str, Any]:
136 """Serialize back into format of puppetfile.yaml."""
137 # data omitted since that gets published as data directory
138 # hiera is ommitted since its published as hiera.yaml
139 # environment name ommited since that becomes the target directory
140 return {'modules': {k: v.serialize() for k, v in self.modules.items()}}
143@dataclass
144class ResolvedPuppetFile(PuppetFile):
145 """
146 Identical to PuppetFile, but modules is the true set.
148 PuppetFile contains the user supplied modules from the
149 puppetfile.yaml, this instead has the true set of modules, where
150 all modules are present, and ALL of them will have exact versions
151 (when needed).
153 It's constructor should be seen assumed private. These objects
154 should only be built through find_all_modules_for_environment.
155 """
157 def publish(self, destination: str) -> None:
158 """
159 Publish actual environments to direcotry.
161 [Parameters]
162 destination - Where to publish to.
163 This is most likely
164 /etc/puppetlabs/code/environments/{env_name}
165 """
166 logger.info('== Building actual environment ==')
167 util.ensure_directory(destination)
168 for module in self.modules.values():
169 logger.info(module)
170 path = os.path.join(destination, 'modules', module.module_path)
171 module.publish(path)
173 if self.hiera:
174 with open(os.path.join(destination, 'hiera.yaml'), 'w') as f:
175 yaml.dump(self.hiera, f)
177 if self.data:
178 for path, data in self.data.items():
179 # Re-normalize path for systems with other path delimiters
180 path = os.path.join(*path.split('/'))
181 path = os.path.join(destination, 'data', f'{path}.yaml')
182 os.makedirs(os.path.dirname(path), exist_ok=True)
183 with open(path, 'w') as f:
184 yaml.dump(data, f)
186 with open(os.path.join(destination, 'puppetfile.yaml'), 'w') as f:
187 yaml.dump(self.serialize(), f)
190def parse_puppetfile(data: dict[str, Any],
191 *,
192 config: Optional[r11k.config.Config] = None,
193 path: Optional[str] = None) -> PuppetFile:
194 """Parse data from puppetfile dictionary."""
195 pf = PuppetFile()
197 if config:
198 pf.config = config
200 for module in data['modules']:
201 if config:
202 module['config'] = config
203 m = parse_module_dict(module)
204 m.explicit = True
205 pf.modules[m.name] = m
207 if env := data.get('environment'): 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true
208 pf.environment_name = env
209 elif path:
210 pf.environment_name, _ = os.path.splitext(os.path.basename(path))
212 if data_entry := data.get('data'): 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true
213 pf.data = data_entry
215 if hiera := data.get('hiera'): 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 pf.hiera = hiera
218 if subpath := data.get('include'):
219 if not path: 219 ↛ 220line 219 didn't jump to line 220, because the condition on line 219 was never true
220 raise ValueError('include only possible when we have a source file')
221 path = os.path.join(os.path.dirname(path), subpath)
222 parent = load_puppetfile(path)
223 pf.include(parent)
225 return pf
228def load_puppetfile(filepath: str) -> PuppetFile:
229 """Load puppetfile.yaml."""
230 with open(filepath) as f:
231 data = yaml.full_load(f)
232 return parse_puppetfile(data, path=filepath)
235def update_module_names(modules: list[PuppetModule]) -> None:
236 """
237 Update module name from metadata.
239 Compare the currently known module name (which probably orginated
240 in the puppetfile) with the name in the modules metadata. Update
241 to the name from the metadata if they differ.
242 """
243 logger.info('== Updating modules (and extracting dependencies) ==')
244 # for every explicit module
245 for module in modules:
246 metadata = module.metadata
247 # Here since we want the "true" name (as per the metadata)
248 if module.name != metadata.name:
249 logger.info('Module and metadata names differ, updating module to match metadata.')
250 logger.info(f'{module.name} ≠ {metadata.name}')
251 module.name = metadata.name
254# TODO figure out proper way to propagate config through everything
255# TODO shouldn't this be a method on PuppetFile?
256def find_all_modules_for_environment(puppetfile: PuppetFile) -> ResolvedPuppetFile:
257 """
258 Find all modules which would make out this puppet environment.
260 Returns a list of modules, including which version they should be.
261 """
262 modules: list[PuppetModule] = list(puppetfile.modules.values())
264 update_module_names(modules)
266 known_modules: dict[str, PuppetModule] = {}
267 for module in modules:
268 known_modules[module.name] = module
270 # Resolve all modules, restarting with the new set of modules
271 # repeatedly until we find a stable point.
272 while True:
273 # Dict from which modules we want, to which versions we can have
274 constraints: dict[str, list[Interval]] = {}
276 # For each module in puppetfile
277 for module in modules:
278 logger.debug(module)
279 # collect its dependencies
280 for depspec in module.metadata.dependencies:
281 # and merge them to the dependency set
282 constraints.setdefault(depspec.name, []) \
283 .append(depspec.interval(by=module.name))
285 # resolve constraints
286 resolved_constraints: dict[str, Interval] = {}
287 for module_name, intervals in constraints.items():
288 # TODO what to do if invalid constraints
289 resolved_constraints[module_name] = reduce(interval.intersect, intervals)
291 # build the next iteration of modules
292 # If this turns out to be identical to modules, then
293 # everything is resolved and we exit. Otherwise we continue
294 # the loop
295 next_modules: dict[str, PuppetModule] = {}
296 for name, interval_ in resolved_constraints.items():
297 if module_ := known_modules.get(name):
298 next_modules[name] = module_
299 else:
300 # TODO keep interval instead of locking it in
301 module_ = ForgePuppetModule(name, config=puppetfile.config)
302 # TODO this crashes when the dependency is a git
303 # module, since it tries to fetch a forge module of
304 # that name.
305 # TODO continue here, fix this
306 module_.version = interval_.newest(module_.versions())
307 next_modules[name] = module_
308 known_modules[name] = module_
310 # TODO what are we even forcing here?
311 for name, module_ in puppetfile.modules.items():
312 if next_modules.get(name): 312 ↛ 313line 312 didn't jump to line 313, because the condition on line 312 was never true
313 logger.warn('Forcing %s', name)
314 next_modules[name] = module_
316 next_modules_list: list[PuppetModule] = list(next_modules.values())
318 if next_modules_list == modules:
319 break
321 modules = next_modules_list
323 return ResolvedPuppetFile(modules={m.name: m for m in modules},
324 environment_name=puppetfile.environment_name,
325 data=puppetfile.data,
326 hiera=puppetfile.hiera,
327 config=puppetfile.config)