diff options
Diffstat (limited to 'definate')
-rw-r--r-- | definate/COPYING | 202 | ||||
-rw-r--r-- | definate/README | 90 | ||||
-rw-r--r-- | definate/__init__.py | 26 | ||||
-rw-r--r-- | definate/definate.yaml | 36 | ||||
-rwxr-xr-x | definate/definition_filter.py | 164 | ||||
-rwxr-xr-x | definate/dns_generator.py | 88 | ||||
-rwxr-xr-x | definate/file_filter.py | 121 | ||||
-rwxr-xr-x | definate/filter_factory.py | 104 | ||||
-rwxr-xr-x | definate/generator.py | 56 | ||||
-rwxr-xr-x | definate/generator_factory.py | 57 | ||||
-rwxr-xr-x | definate/global_filter.py | 68 | ||||
-rwxr-xr-x | definate/yaml_validator.py | 87 |
12 files changed, 1099 insertions, 0 deletions
diff --git a/definate/COPYING b/definate/COPYING new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/definate/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/definate/README b/definate/README new file mode 100644 index 0000000..5c0823f --- /dev/null +++ b/definate/README @@ -0,0 +1,90 @@ +Definate is part of Capirca and is a system to develop and manage network +definitions that can be used in Capirca policies. +It was developed by Google for internal use, and is now open source. + +Project home page: http://code.google.com/p/capirca/ + +Please send contributions to capirca-dev@googlegroups.com. + +Code should include unit tests and follow the Google Python style guide: +http://code.google.com/p/soc/wiki/PythonStyleGuide + +================================================================================ + +Definate Configuration File + +Global section + This section contains directives global in scope. + + [pre|post]_filters: Optional list of global filters to run before + (pre_filters) or after (post_filters) the definition file generation. + Each filter may define a set of filter specific arguments in the 'args' + attribute. + 'name': Name of the filter used to lookup the filter class to use. + Valid values: There are currently no implementations. + per_file_[pre|post]_filters: Optional list of file filters run before + (pre_filters) or after (post_filters) the generation of each file. + The pre_filters here are run BEFORE the individual filters specified in + the files section and the post_filters are run AFTER the individual + filters specified in the files section. + For a list of possible filters and arguments, see the "Files section". + per_definition_[pre|post]_filters: Optional list of definition filters run + before (pre_filters) or after (post_filters) the generation of each + definition. + The pre_filters here are run BEFORE the individual filters specified in + the definitions section and the post_filters are run AFTER the individual + filters specified in the definitions section. + For a list of possible filters and arguments, see the "Definitions + section". + +Files section + This section contains a list of settings and configurations for each + file that gets generated. + + path: Path to the definitions file to be generated, relative to the + def path defined in the global section. + [pre|post]_filters: Optional list of file level filters to run before + (pre_filters) or after (post_filters) the file has been generated. + Each filter may define a set of filter specific arguments in the + 'args' attribute. + 'name': Name of the filter used to lookup the filter class to use. + Valid values: + 'PrintFilter': Does not modify input. Just prints it. + 'WriteFileFilter': Writes files out locally. + file_header: List of strings that get printed in the beginning of the file. + generators: List of generator blocks. + +Generators section + name: The generator defines the source of the information. + Valid values: + 'DnsGenerator': For definitions generated based on hostnames with a simple + DNS resolver. Note that the resolver might not return all addresses used + for one hostname depending on the implemented DNS load balancing. + definitions: List of definition blocks. + +Definitions section + name: Name of the definition that gets generated. This name is used in the + definitions file and can be used in policies to reference the definition. + header: Optional list of header strings that will be printed before the + definition. + [pre|post]_filters: Optional list of definition level filters to run + before (pre_filters) or after (post_filters) the definition has been + generated. Each filter may define a set of filter specific arguments + in the 'args' attribute. + 'name': Name of the filter used to lookup the filter class to use. + Valid values: + 'SortFilter': Sort the input list and return one list containing + IPv4 sorted, IPv6 sorted. + 'AlignFilter': Take the input list and definition name and create + nicely formated output. + networks: Contains a list of descriptions about how to get a complete set + of networks/IPs for one definition. This section contains + generator-specific configuration directives". + +Network directives for DnsGenerator + +Networks section + names: List of hostnames that should be resolved. + types: List of types the output should be filtered for. Valid values: + 'A': Filter for IPv4 addresses. + 'AAAA': Filter for IPv6 addresses. diff --git a/definate/__init__.py b/definate/__init__.py new file mode 100644 index 0000000..e6069ec --- /dev/null +++ b/definate/__init__.py @@ -0,0 +1,26 @@ +# +# Network definition generator libraries +# +# definate/__init__.py +# +# This package is intended to provide functionality to generate lists of network +# definitions that can be used within other network definitions and policies of +# Capirca. +# +# from definate import generator +# from definate import generator_factory +# from definate import dns_generator +# from definate import filter_factory +# from definate import global_filter +# from definate import file_filter +# from definate import definition_filter +# from definate import yaml_validator +# + +__version__ = '1.0.0' + +__all__ = ['generator', 'generator_factory', 'dns_generator', + 'filter_factory', 'global_filter', 'file_filter', + 'definition_filter', 'yaml_validator'] + +__author__ = 'Martin Suess (msu@google.com)' diff --git a/definate/definate.yaml b/definate/definate.yaml new file mode 100644 index 0000000..9e9690e --- /dev/null +++ b/definate/definate.yaml @@ -0,0 +1,36 @@ +# Definate configuration +# For usage information, see README file. +global: + per_file_post_filters: + - name: 'WriteFileFilter' + per_definition_post_filters: + - name: 'SortFilter' + - name: 'AlignFilter' +files: + - path: 'AUTOGEN.net' + file_header: + - 'This file is autogenerated. Please do not edit it manually.' + - 'Instead run Definate: ./definate.py' + generators: + - name: 'DnsGenerator' + definitions: + - name: 'WWW_AUTOGEN' + header: + - 'WWW Clusters' + - 'Generated from DNS names (best effort)' + networks: + - names: + - 'www.google.com' + - 'www.gmail.com' + types: ['A', 'AAAA'] + - name: 'NS_AUTOGEN' + header: + - 'NS Clusters' + - 'Generated from DNS names (best effort)' + networks: + - names: + - 'ns1.google.com' + - 'ns2.google.com' + - 'ns3.google.com' + - 'ns4.google.com' + types: ['A', 'AAAA'] diff --git a/definate/definition_filter.py b/definate/definition_filter.py new file mode 100755 index 0000000..492afc7 --- /dev/null +++ b/definate/definition_filter.py @@ -0,0 +1,164 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module that holds all definition-level filter classes of Definate.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import logging + + +class Container(object): + """Container class to hold all information to be passed between filters.""" + + def __init__(self, header=None, name='', entries_and_comments=None, + string_representation=''): + """Initializer. + + Args: + header: Optional list of strings to be added as headers. + name: Optional string representing the name of the definition. + entries_and_comments: Optional list of tuples (entries, comments) which + hold all entries for one definition as well as comments. + string_representation: Optional string holding the string representation + of the definition (typically used as output e.g. in a file in the end). + """ + self.header = header if header else [] + self.name = name + self.entries_and_comments = ( + entries_and_comments if entries_and_comments else []) + self.string_representation = string_representation + + +class DefinitionFilter(object): + """Abstract class defining the interface for the filter chain objects.""" + + def Filter(self, container, args): + """Interface to filter or modify data passed into it. + + Args: + container: Container object which holds all information for one + definition. See Container class for details. + args: Dictionary of arguments depending on the actual filter in use. + + Raises: + NotImplementedError: In any case since this is not implemented an needs + to be defined by subclasses. + """ + raise NotImplementedError( + 'This is an interface only. Implemented by subclasses.') + + +class SortFilter(DefinitionFilter): + """DefinitionFilter implementation which sorts all entries for nice output.""" + + def Filter(self, container, unused_args): + """Filter method that sorts all entries in a definition for nice output. + + The filter sorts all entries in ascending order: + - IPv4 networks + - IPv6 networks + + Args: + container: Container object which holds all information for one + definition. See Container class for details. + unused_args: No extra arguments required by this filter implementation. + + Returns: + Container object that has been passed in. + """ + ipv4_nodes = [] + ipv6_nodes = [] + + for node, comment in container.entries_and_comments: + if node.version == 4: + ipv4_nodes.append((node, comment)) + elif node.version == 6: + ipv6_nodes.append((node, comment)) + else: + logging.warn('Unsupported address version detected: %s', node.version) + + ipv4_nodes = self._RemoveDuplicateNetworks(ipv4_nodes) + ipv6_nodes = self._RemoveDuplicateNetworks(ipv6_nodes) + + ipv4_nodes.sort() + ipv6_nodes.sort() + + container.entries_and_comments = ipv4_nodes + ipv6_nodes + return container + + def _RemoveDuplicateNetworks(self, network_list): + """Method to remove duplicate networks from the network list. + + Args: + network_list: List of node/comment tuples where node is an IPNetwork + object and comment is a string. + + Returns: + The same list of networks and comments minus duplicate entries. + """ + result_list = [] + result_dict = {} + for node, comment in network_list: + result_dict[str(node)] = (node, comment) + for node in result_dict: + result_list.append(result_dict[node]) + return result_list + + +class AlignFilter(DefinitionFilter): + """DefinitionFilter implementation which generates nicely aligned output.""" + + def Filter(self, container, unused_args): + """Filter method that aligns the entries in the output nicely. + + This code formats the entries_and_comments by figuring out the + left-justification from the definition name ('name'), and padding the + left justification of the comments to 3 spaces after the longest entry + length. + + In order to do this succinctly, without adding strings together, we use a + format string that we replace twice. Once for the (left|right) + justification bounds, and again with the final values. + + Args: + container: Container object which holds all information for one + definition. See Container class for details. + unused_args: No extra arguments required by this filter implementation. + + Returns: + Container object that has been passed in. + """ + first_format_string = '%%s = %%%is# %%s' + format_string = '%%%is%%%is# %%s' + + max_len = max(len(str(e)) for e, _ in container.entries_and_comments) + value_justification = -1 * (max_len + 3) + column_justification = len(container.name) + 3 # 3 for ' = ' + + first_format_string %= value_justification + format_string %= (column_justification, value_justification) + + entry, comment = container.entries_and_comments[0] + first_string = first_format_string % (container.name, entry, comment) + output = [first_string] + + for entry, comment in container.entries_and_comments[1:]: + output.append(format_string % ('', entry, comment)) + + container.string_representation = '\n'.join(output) + return container diff --git a/definate/dns_generator.py b/definate/dns_generator.py new file mode 100755 index 0000000..bb17c71 --- /dev/null +++ b/definate/dns_generator.py @@ -0,0 +1,88 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generator for DNS based network definitions.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import logging +import socket + +from third_party import ipaddr +import generator + + +class DnsGeneratorError(Exception): + """Exception to use when DnsGenerator fails.""" + + +class DnsGenerator(generator.Generator): + """Generator implementation for network definitions based on DNS.""" + + SUPPORTED_TYPES = ['A', 'AAAA'] + + def GenerateDefinition(self, config, unused_global_config): + """Generates a list of all nodes in a network definition. + + This method basically processes all the configuration which is + hierarchically below "networks" in the "definitions" section in the + configuration file to generate a list of all nodes in that definition. + + Args: + config: YAML configuration structure (dictionaries, lists and strings) + representing the "networks" section in "definitions" of the + configuration file. + unused_global_config: YAML configuration structure (dictionaries, lists + and strings) representing the "global" section of the configuration + file. + + Returns: + Tuples of IPNetwork objects and string comments representing all the nodes + in one definition. + + Raises: + DefinateConfigError: The configuration is not well formed. + DnsGeneratorError: There is a problem generating the output. + """ + nodes = [] + yaml_structure = { + 'names': ['str'], + 'types': ['str'], + } + for network in config: + self._yaml_validator.CheckConfiguration(network, yaml_structure) + for typ in network['types']: + if typ not in self.SUPPORTED_TYPES: + raise DnsGeneratorError('Unsupported DNS type found: %s' % typ) + for name in network['names']: + try: + addr_list = socket.getaddrinfo(name, None) + except socket.gaierror: + raise DnsGeneratorError('Hostname not found: %s' % name) + for family, _, _, _, sockaddr in addr_list: + ip_addr = None + if family == socket.AF_INET and 'A' in network['types']: + # sockaddr = (address, port) + ip_addr = ipaddr.IPv4Network(sockaddr[0]) + elif family == socket.AF_INET6 and 'AAAA' in network['types']: + # sockaddr = (address, port, flow info, scope id) + ip_addr = ipaddr.IPv6Network(sockaddr[0]) + else: + logging.debug('Skipping unknown AF \'%d\' for: %s', family, name) + if ip_addr: + nodes.append((ip_addr, name)) + return nodes diff --git a/definate/file_filter.py b/definate/file_filter.py new file mode 100755 index 0000000..021cc09 --- /dev/null +++ b/definate/file_filter.py @@ -0,0 +1,121 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module that holds all file-level filter classes of Definate.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import logging + + +class Error(Exception): + """Base error class.""" + + +class FileError(Error): + """Exception to use when file handling files.""" + + +class Container(object): + """Container class to hold all information to be passed between filters.""" + + def __init__(self, lines=None, relative_path='', absolute_path=''): + """Initializer. + + Args: + lines: Optional list of strings which will be added as lines. + E.g. the file header can be added here directly. + relative_path: Optional string to specify the path of the file relative to + the location of the definition directory (e.g. 'AUTOGEN.net'). + absolute_path: Optional string to specify the absolute path of the local + file to be written (e.g. '/tmp/AUTOGEN.net') or if a SCM software is + used it can refer to the full path there + (e.g. '//depot/def/AUTOGEN.net'). + """ + self.lines = lines if lines else [] + self.absolute_path = absolute_path + self.relative_path = relative_path + + +class FileFilter(object): + """Abstract class defining the interface for the filter chain objects.""" + + def Filter(self, container, args): + """Interface to filter or modify data passed into it. + + Args: + container: Container object which holds all information for one definition + file. See Container class for details. + args: Dictionary of arguments depending on the actual filter in use. + + Raises: + NotImplementedError: In any case since this is not implemented an needs + to be defined by subclasses. + """ + raise NotImplementedError( + 'This is an interface only. Implemented by subclasses.') + + +class PrintFilter(FileFilter): + """FileFilter implementation which simply logs the file content.""" + + def Filter(self, container, unused_args): + """Filter method that prints the content of the file to stdout. + + Args: + container: Container object which holds all information for one definition + file. See Container class for details. + unused_args: No extra arguments required by this filter implementation. + + Returns: + Container object that has been passed in. + """ + print '# File "%s"' % container.absolute_path + print '\n'.join(container.lines) + return container + + +class WriteFileFilter(FileFilter): + """FileFilter implementation which writes the content into a file.""" + + def Filter(self, container, unused_args): + """Filter method that writes the content of the file into a file. + + Args: + container: Container object which holds all information for one definition + file. See Container class for details. + unused_args: No extra arguments required by this filter implementation. + + Returns: + Container object that has been passed in. + """ + try: + f = file(container.absolute_path, 'w') + except IOError as e: + raise FileError('File "%s" could not be opened: %s' % ( + container.absolute_path, e)) + + try: + f.write('\n'.join(container.lines)) + except IOError as e: + raise FileError('File "%s" could not be written: %s' % ( + container.absolute_path, e)) + else: + f.close() + + logging.info('Wrote file: %s', container.absolute_path) + return container diff --git a/definate/filter_factory.py b/definate/filter_factory.py new file mode 100755 index 0000000..1ad1d3f --- /dev/null +++ b/definate/filter_factory.py @@ -0,0 +1,104 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functionality to allow easily retrieving certain filter objects.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import definition_filter +import file_filter + +DEFINITION_FILTER = 1 +FILE_FILTER = 2 +GLOBAL_FILTER = 3 + +PRE_FILTERS = 'PreFilters' +POST_FILTERS = 'PostFilters' + + +class Error(Exception): + """Base error class.""" + + +class FilterIdentificationError(Error): + """Exception to use when FilterFactory fails to identify the Filter.""" + + +class FilterFactory(object): + """Functionality to get a filter object easily based on its name. + + This class can be initialized and the GetFilter() method allows retrieving a + specific filter based on the name of the filter and the scope (global, file + and definition). + """ + + def __init__(self): + """Initializer.""" + self._filters = { + DEFINITION_FILTER: { + 'PostFilters': { + 'SortFilter': definition_filter.SortFilter, + 'AlignFilter': definition_filter.AlignFilter, + }, + }, + FILE_FILTER: { + 'PostFilters': { + 'PrintFilter': file_filter.PrintFilter, + 'WriteFileFilter': file_filter.WriteFileFilter, + }, + }, + GLOBAL_FILTER: { + 'PreFilters': { + }, + 'PostFilters': { + }, + }, + } + + def GetFilter(self, scope, identifier, sequence): + """Returns a specific filter instance based on the identifier. + + Args: + scope: Type of filter to be returned. Valid types are listed as globals + in the beginning of this module. + identifier: String identifier for the filter to get. + sequence: String identifier for the sequence information to determine + when the filter should be applied. Valid values: + - 'PreFilters': Filters that are applied before processing the data + (e.g. before the definition is created). + - 'PostFilters': Filters that are applied after processing the data + (e.g. after the definition has been created). + + Raises: + FilterIdentificationError: If the filter cannot be identified. + + Returns: + Filter instance based on the identifier passed in. + """ + if scope not in self._filters: + raise FilterIdentificationError( + 'Filter scope \'%d\' could not be found in filters.' % scope) + if sequence not in self._filters[scope]: + raise FilterIdentificationError( + 'Filter sequence \'%s\' is not applicable to scope \'%d\'.' % ( + sequence, scope)) + filters = self._filters[scope][sequence] + if identifier not in filters: + raise FilterIdentificationError( + 'Filter \'%s\' could not be identified. Wrong scope (%d) or sequence' + ' (%s)?' % (identifier, scope, sequence)) + return filters[identifier]() diff --git a/definate/generator.py b/definate/generator.py new file mode 100755 index 0000000..22cad08 --- /dev/null +++ b/definate/generator.py @@ -0,0 +1,56 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module holding the abstract definition generator class.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import yaml_validator + + +class Error(Exception): + """Base error class.""" + + +class GeneratorError(Error): + """Base Generator error class to inherit from in specific generators.""" + + +class Generator(object): + """Abstract class defining the interface for the definition generation.""" + + def __init__(self): + """Initializer.""" + self._yaml_validator = yaml_validator.YamlValidator() + + def GenerateDefinition(self, config, global_config): + """Interface to generate definitions based on a configuration passed in. + + Classes inheriting from Generator should implement this interface by parsing + the configuration and generating a network definition based on it. + For reference, have a look at the already implemented classes. + + Args: + config: Configuration necessary to generate one full definition. + global_config: Global configuration section. + + Raises: + NotImplementedError: In any case since this is not implemented and needs + to be defined by sublcasses. + """ + raise NotImplementedError( + 'This is an interface only. Implemented by subclasses.') diff --git a/definate/generator_factory.py b/definate/generator_factory.py new file mode 100755 index 0000000..c887cc1 --- /dev/null +++ b/definate/generator_factory.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functionality to allow easily retrieving the right definition generator.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import dns_generator + + +class Error(Exception): + """Base error class.""" + + +class GeneratorIdentificationError(Error): + """Exception to use when GeneratorFactory fails to identify the Generator.""" + + +class GeneratorFactory(object): + """Functionality to get a definition generator easily based on its name.""" + + def __init__(self): + """Initializer.""" + self._generators = { + 'DnsGenerator': dns_generator.DnsGenerator, + } + + def GetGenerator(self, identifier): + """Returns a specific generator instance based on the identifier. + + Args: + identifier: String identifier for the generator to get. + + Raises: + GeneratorIdentificationError: If the generator cannot be identified. + + Returns: + Generator instance based on the identifier passed in. + """ + if identifier not in self._generators: + raise GeneratorIdentificationError( + 'Generator \'%s\' could not be identified.' % identifier) + return self._generators[identifier]() diff --git a/definate/global_filter.py b/definate/global_filter.py new file mode 100755 index 0000000..5969443 --- /dev/null +++ b/definate/global_filter.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module that holds all global-level filter classes of Definate.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +import yaml_validator + + +class Error(Exception): + """Base error class.""" + + +class Container(object): + """Container class to hold all information to be passed between filters.""" + + def __init__(self, absolute_paths=None, relative_paths=None): + """Initializer. + + Args: + absolute_paths: Optional list of strings to specify the full path of the + generated files + (e.g. ['//depot/def/AUTOGEN1.net', '/tmp/AUTOGEN2.net']). + relative_paths: Optional list of strings to specify the paths of the + generated files relative to the location of the definition directory + (e.g. ['AUTOGEN1.net']). + """ + self.absolute_paths = absolute_paths if absolute_paths else [] + self.relative_paths = relative_paths if relative_paths else [] + self.changelist = '' + + +class GlobalFilter(object): + """Abstract class defining the interface for the filter chain objects.""" + + def __init__(self): + """Initializer.""" + self._yaml_validator = yaml_validator.YamlValidator() + + def Filter(self, container, args): + """Interface to filter or modify data passed into it. + + Args: + container: Container object which holds all global information. + See Container class for details. + args: Dictionary of arguments depending on the actual filter in use. + + Raises: + NotImplementedError: In any case since this is not implemented an needs + to be defined by subclasses. + """ + raise NotImplementedError( + 'This is an interface only. Implemented by subclasses.') diff --git a/definate/yaml_validator.py b/definate/yaml_validator.py new file mode 100755 index 0000000..5aae74a --- /dev/null +++ b/definate/yaml_validator.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +# +# Copyright 2012 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools to allow the verification of the YAML configuration for Definate.""" + +__author__ = 'msu@google.com (Martin Suess)' + + +class Error(Exception): + """Base error class.""" + + +class DefinateConfigError(Error): + """Exception to use when Definate fails reading the configuration.""" + + +class YamlValidator(object): + """Class to verify the sanity of a YAML configuration.""" + + def CheckConfigurationItem(self, dictionary, item, typ=None): + """Checks for the presence of an item in a dictionary. + + Args: + dictionary: Configuration part that should be checked. + item: Name of the key to check as string. + typ: Type of the value to check. Default is to not check the type. + + Raises: + DefinateConfigError: The configuration is not sane. + """ + if not dictionary or item not in dictionary: + raise DefinateConfigError('"%s" is not defined in config: %s' + % (item, dictionary)) + if typ and type(dictionary[item]) is not typ: + raise DefinateConfigError('Type of "%s" is %s, expected %s.' % + (item, str(type(dictionary[item])), str(typ))) + + def CheckConfiguration(self, config, structure, max_recursion_depth=30): + """Recursively checks the sanity and structure of the configuration. + + This method checks the sanity of the configuration structure for Definate + and raises a DefinateConfigError if the configuration is not sane. + + Args: + config: Dictionary generated from the YAML configuration file which should + be checked. + structure: Structure of the configuration against which should be checked. + max_recursion_depth: Defines the maximum amount of recursion cycles before + checking is aborted. Default is 30. + + Raises: + DefinateConfigError: The configuration is not sane. + """ + max_depth = max_recursion_depth - 1 + if max_depth <= 0: + raise DefinateConfigError('Maximum recursion depth reached. Please check ' + 'configuration manually.') + if type(structure) in [dict, list]: + for item in structure: + value = item + if type(structure) is dict: + value = structure[item] + + self.CheckConfigurationItem(config, item, typ=type(value)) + if type(value) is dict: + self.CheckConfiguration(config[item], value, max_depth) + elif type(value) is list: + for (i, list_value) in enumerate(value): + self.CheckConfiguration(config[item][i], list_value, max_depth) + elif type(structure) is type(config): + return + else: + raise DefinateConfigError('Type of "%s" is %s, expected %s.' % ( + config, str(type(config)), str(structure))) |