Source code for Bcfg2.Server.Lint.Comments

""" Check files for various required comments. """

import os
import lxml.etree
import Bcfg2.Options
import Bcfg2.Server.Lint
from Bcfg2.Server import XI_NAMESPACE
from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \
    import CfgPlaintextGenerator
from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator
from Bcfg2.Server.Plugins.Cfg.CfgJinja2Generator import CfgJinja2Generator
from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML


[docs]class Comments(Bcfg2.Server.Lint.ServerPlugin): """ The Comments lint plugin checks files for header comments that give information about the files. For instance, you can require SVN keywords in a comment, or require the name of the maintainer of a Genshi template, and so on. """ options = Bcfg2.Server.Lint.ServerPlugin.options + [ Bcfg2.Options.Option( cf=("Comments", "global_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for all file types"), Bcfg2.Options.Option( cf=("Comments", "global_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for all file types"), Bcfg2.Options.Option( cf=("Comments", "bundler_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for non-templated bundles"), Bcfg2.Options.Option( cf=("Comments", "bundler_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for non-templated bundles"), Bcfg2.Options.Option( cf=("Comments", "genshibundler_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for templated bundles"), Bcfg2.Options.Option( cf=("Comments", "genshibundler_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for templated bundles"), Bcfg2.Options.Option( cf=("Comments", "properties_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for Properties files"), Bcfg2.Options.Option( cf=("Comments", "properties_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Properties files"), Bcfg2.Options.Option( cf=("Comments", "cfg_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for non-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "cfg_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for non-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "genshi_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for Genshi-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "genshi_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Genshi-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "cheetah_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for Cheetah-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "cheetah_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Cheetah-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "jinja2_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for Jinja2-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "jinja2_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for Jinja2-templated Cfg files"), Bcfg2.Options.Option( cf=("Comments", "infoxml_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for info.xml files"), Bcfg2.Options.Option( cf=("Comments", "infoxml_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for info.xml files"), Bcfg2.Options.Option( cf=("Comments", "probes_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for probes"), Bcfg2.Options.Option( cf=("Comments", "probes_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for probes"), Bcfg2.Options.Option( cf=("Comments", "metadata_keywords"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required keywords for metadata files"), Bcfg2.Options.Option( cf=("Comments", "metadata_comments"), type=Bcfg2.Options.Types.comma_list, default=[], help="Required comments for metadata files")] def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.config_cache = {}
[docs] def Run(self): self.check_bundles() self.check_properties() self.check_metadata() self.check_cfg() self.check_probes()
[docs] @classmethod def Errors(cls): return {"unexpanded-keywords": "warning", "keywords-not-found": "warning", "comments-not-found": "warning", "broken-xinclude-chain": "warning"}
[docs] def required_keywords(self, rtype): """ Given a file type, fetch the list of required VCS keywords from the bcfg2-lint config. Valid file types are documented in :manpage:`bcfg2-lint.conf(5)`. :param rtype: The file type :type rtype: string :returns: list - the required items """ return self.required_items(rtype, "keyword")
[docs] def required_comments(self, rtype): """ Given a file type, fetch the list of required comments from the bcfg2-lint config. Valid file types are documented in :manpage:`bcfg2-lint.conf(5)`. :param rtype: The file type :type rtype: string :returns: list - the required items """ return self.required_items(rtype, "comment")
[docs] def required_items(self, rtype, itype): """ Given a file type and item type (``comment`` or ``keyword``), fetch the list of required items from the bcfg2-lint config. Valid file types are documented in :manpage:`bcfg2-lint.conf(5)`. :param rtype: The file type :type rtype: string :param itype: The item type (``comment`` or ``keyword``) :type itype: string :returns: list - the required items """ if itype not in self.config_cache: self.config_cache[itype] = {} if rtype not in self.config_cache[itype]: rv = [] rv.extend(getattr(Bcfg2.Options.setup, "global_%ss" % itype)) local_reqs = getattr(Bcfg2.Options.setup, "%s_%ss" % (rtype.lower(), itype)) if local_reqs == ['']: # explicitly specified as empty rv = [] else: rv.extend(local_reqs) self.config_cache[itype][rtype] = rv return self.config_cache[itype][rtype]
[docs] def check_bundles(self): """ Check bundle files for required comments. """ if 'Bundler' in self.core.plugins: for bundle in self.core.plugins['Bundler'].entries.values(): xdata = None rtype = "" try: xdata = lxml.etree.XML(bundle.data) rtype = "bundler" except (lxml.etree.XMLSyntaxError, AttributeError): xdata = \ lxml.etree.parse(bundle.template.filepath).getroot() rtype = "genshibundler" self.check_xml(bundle.name, xdata, rtype)
[docs] def check_properties(self): """ Check Properties files for required comments. """ if 'Properties' in self.core.plugins: props = self.core.plugins['Properties'] for propfile, pdata in props.entries.items(): if os.path.splitext(propfile)[1] == ".xml": self.check_xml(pdata.name, pdata.xdata, 'properties')
[docs] def has_all_xincludes(self, mfile): """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files` includes all XIncludes listed in the specified metadata type, false otherwise. In other words, this returns True if bcfg2-lint is dealing with complete metadata. :param mfile: The metadata file ("clients.xml" or "groups.xml") to check for XIncludes :type mfile: string :returns: bool """ if self.files is None: return True else: path = os.path.join(self.metadata.data, mfile) if path in self.files: xdata = lxml.etree.parse(path) for el in xdata.findall('./%sinclude' % XI_NAMESPACE): if not self.has_all_xincludes(el.get('href')): self.LintError("broken-xinclude-chain", "Broken XInclude chain: could not " "include %s" % path) return False return True
[docs] def check_metadata(self): """ Check Metadata files for required comments. """ if self.has_all_xincludes("groups.xml"): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, "metadata") if hasattr(self.metadata, "clients_xml"): if self.has_all_xincludes("clients.xml"): self.check_xml(os.path.join(self.metadata.data, "clients.xml"), self.metadata.clients_xml.data, "metadata")
[docs] def check_cfg(self): """ Check Cfg files and ``info.xml`` files for required comments. """ if 'Cfg' in self.core.plugins: for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): rtype = None if isinstance(entry, CfgGenshiGenerator): rtype = "genshi" elif isinstance(entry, CfgPlaintextGenerator): rtype = "cfg" elif isinstance(entry, CfgCheetahGenerator): rtype = "cheetah" elif isinstance(entry, CfgJinja2Generator): rtype = "jinja2" elif isinstance(entry, CfgInfoXML): self.check_xml(entry.infoxml.name, entry.infoxml.xdata, "infoxml") continue if rtype: self.check_plaintext(entry.name, entry.data, rtype)
[docs] def check_probes(self): """ Check Probes for required comments """ if 'Probes' in self.core.plugins: for probe in self.core.plugins['Probes'].probes.entries.values(): self.check_plaintext(probe.name, probe.data, "probes")
[docs] def check_xml(self, filename, xdata, rtype): """ Generic check to check an XML file for required comments. :param filename: The filename :type filename: string :param xdata: The file data :type xdata: lxml.etree._Element :param rtype: The type of file. Available types are documented in :manpage:`bcfg2-lint.conf(5)`. :type rtype: string """ self.check_lines(filename, [str(el) for el in xdata.getiterator(lxml.etree.Comment)], rtype)
[docs] def check_plaintext(self, filename, data, rtype): """ Generic check to check a plain text file for required comments. :param filename: The filename :type filename: string :param data: The file data :type data: string :param rtype: The type of file. Available types are documented in :manpage:`bcfg2-lint.conf(5)`. :type rtype: string """ self.check_lines(filename, data.splitlines(), rtype)
[docs] def check_lines(self, filename, lines, rtype): """ Generic header check for a set of lines. :param filename: The filename :type filename: string :param lines: The data to check :type lines: list of strings :param rtype: The type of file. Available types are documented in :manpage:`bcfg2-lint.conf(5)`. :type rtype: string """ if self.HandlesFile(filename): # found is trivalent: # False == keyword not found # None == keyword found but not expanded # True == keyword found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) for line in lines: # we check for both '$<keyword>:' and '$<keyword>$' to see # if the keyword just hasn't been expanded for (keyword, status) in found.items(): if not status: if '$%s:' % keyword in line: found[keyword] = True elif '$%s$' % keyword in line: found[keyword] = None unexpanded = [keyword for (keyword, status) in found.items() if status is None] if unexpanded: self.LintError("unexpanded-keywords", "%s: Required keywords(s) found but not " "expanded: %s" % (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] if missing: self.LintError("keywords-not-found", "%s: Required keywords(s) not found: $%s$" % (filename, "$, $".join(missing))) # next, check for required comments. found is just # boolean found = dict((k, False) for k in self.required_comments(rtype)) for line in lines: for (comment, status) in found.items(): if not status: found[comment] = comment in line missing = [comment for (comment, status) in found.items() if status is False] if missing: self.LintError("comments-not-found", "%s: Required comments(s) not found: %s" % (filename, ", ".join(missing)))