One Hat Cyber Team

View File Name : plugin_manager.py

from agent_util import Plugin
import logging
import os
import sys
import traceback
import tempfile
import tarfile
import zipfile
try:
    from hashlib import sha1
except:
    from sha import sha as sha1
try:
    # Python 2.x
    import urllib2
except:
    import urllib.request as urllib2
# In case of python 3
try:
    import ConfigParser as configparser
except:
    import configparser

def execfile3(filepath, globals=None, locals=None):
    if globals is None:
        globals = {}
    globals.update({
        "__file__": filepath,
        "__name__": "__main__",
    })
    f = open(filepath, 'rb')
    exec(compile(f.read(), filepath, 'exec'), globals, locals)
    f.close()


class PluginManager(object):
    def __init__(self, db, config_file, *directories):
        self.log = logging.getLogger(self.__class__.__name__)
        self.db = db
        self.config_file = config_file
        self.directories = directories
        self.plugins = self._load_plugins(directories)

        # Internal container for metadata so we can avoid repeatedly regenerating it
        self._metadata = None
        self._hashed_metadata = None

        # now we load all the plugin-specific configs and map them to
        # self.config
        self.config = {}
        if os.path.exists(self.config_file):
            config_file = configparser.ConfigParser()
            config_file.read(self.config_file)
            for textkey in config_file.sections():
                try: debug = config_file.get('agent', 'debug')
                except: debug = False
                self.config[textkey] = {'debug': debug}
                for k, v in config_file.items(textkey):
                    self.config[textkey][k] = v

    def add_dem_wifi_results(self, wifi_results):
        dem_config = self.config.get('dem', None)
        if dem_config:
            dem_config['wifi_results'] = wifi_results

    def install_remote_plugins(self, url):
        """
        Grab the remote url and extract the contents, putting them in the
        appropiate plugin folder.
        """
        base_custom_plugin_dir = self.directories[-1]
        f = tempfile.NamedTemporaryFile(delete=False)
        num_installed = 0
        ext = url.split(".")[-1]
        fname = url.split("/")[-1]
        try:
            r = urllib2.urlopen(url)
            if "content-disposition" in r.info():
                ext = r.info().getheader("content-disposition").split(".")[-1]
            f.write(r.read())
            f.close()
        except Exception:
            self.log.error("Unable to download URL: %s" % traceback.format_exc())
            return

        if ext in ("tar", "tgz"):
            try:
                t = tarfile.open(f.name)
                for file in t.getnames():
                    if file.endswith(".py"):
                        self.log.info("  Installing plugin %s" % file)
                        t.extract(file, base_custom_plugin_dir)
                        num_installed += 1
            except Exception:
                self.log.error("Unable to extract tar contents: %s" % traceback.format_exc())
        elif ext == "zip":
            try:
                z = zipfile.ZipFile(f.name)
                for file in z.namelist():
                    if file.endswith(".py"):
                        self.log.info("  Installing plugin %s" % file)
                        z.extract(file, base_custom_plugin_dir)
                        num_installed += 1
            except Exception:
                self.log.error("Unable to extract zip contents: %s" % traceback.format_exc())
        elif ext == "py":
            self.log.info("  Installing plugin %s" % fname)
            os.system("cp %s %s" % (f.name, os.path.join(base_custom_plugin_dir, fname)))
            num_installed += 1
        else:
            self.log.error("Unable to install Agent Plugin, unknown extension: %s" % ext)

        if num_installed:
            self.log.info("\nInstalled %s Agent plugins" % num_installed)

        os.system("rm -f %s" % f.name)

    # run a specific plugin
    def check(self, schedule):
        try:
            self_key = self.config.get('agent', {}).get('server_key')
            container = None
            if getattr(schedule, 'server_key', None) and schedule.server_key != self_key:
                container_id = schedule.server_key.split(":")[-1]
                try:
                    container = self.db["docker_containers"][container_id]
                except:
                    if schedule.plugin_textkey == "docker" \
                            and schedule.resource_textkey == "status.running":
                        # Special handling of "container is running" metrics.
                        # Pass in a phony value to treat it as gone.
                        container = {"State": "deleted"}
                    else:
                        self.log.error("Could not find container with id %s", container_id)
                        return None

            plugin = self.plugins[schedule.plugin_textkey](schedule)
            config = self.config.get(schedule.plugin_textkey, {})

            scale = config.get('scale', 1.0)

            if container:
                value = plugin.check_docker(container, schedule.resource_textkey, schedule.option, config)
            else:
                value = plugin.check(schedule.resource_textkey, schedule.option, config)
            
            if value is not None: value *= scale

        except Exception:
            self.log.exception(
                '%s/%s check failed: %s' % 
                (schedule.plugin_textkey, schedule.resource_textkey, traceback.format_exc())
            )
            return None

        return value

    def _load_plugins(self, directories):
        plugins = {}
        plugin_blacklist = ''

        if os.path.exists(self.config_file):
            config_file = configparser.ConfigParser()
            config_file.read(self.config_file)

            try:
                plugin_blacklist = config_file.get('agent', 'plugin_blacklist')
            except:
                plugin_blacklist = ''

            if plugin_blacklist:
                plugin_blacklist = list(map(str.strip, plugin_blacklist.split(',')))

        def plugin_file_checker():
            for plugin_dir in directories:
                self.log.info('Looking in directory %r', plugin_dir)
                for name in os.listdir(plugin_dir):
                    try:
                        if name[-3:] != '.py' or name == '__init__.py':
                            continue

                        plugin_file = os.path.join(plugin_dir, name)

                        # We've registed sys.modules['agent'] to equal THIS
                        # module, so this exec has the effect of 'importing' the
                        # plugin, and resolving 'import agent' to import this
                        # module. This is so we never have to change the plugins
                        # in order to whitelabel.
                        f_globals = {}
                        f_locals = {}
                        if sys.version_info[0] == 3:
                            execfile3(plugin_file, f_globals, f_globals)
                        else:
                            execfile(plugin_file, f_globals, f_globals)

                        for obj in list(f_globals.values()):
                            yield obj
                        self.log.debug('Loaded %r', plugin_file)

                    except Exception:
                        t, e = sys.exc_info()[:2]
                        self.log.info('Error importing plugin %s, skipping',
                                           name)
                        self.log.debug(e)
                        continue
                self.log.info('Loaded %d plugins', len(plugins))

        for obj in plugin_file_checker():
            if isinstance(obj, type):
                if issubclass(obj, Plugin) and hasattr(obj, 'textkey') and obj.textkey not in plugin_blacklist:
                    if obj.textkey != "TEMPLATE_PLUGIN":
                        plugins[obj.textkey] = obj

        return plugins

    # returns all the values needed to describe the plugins and their resources
    # in a json-serializable way.  Uses local cache of values to avoid recomputing.
    @property
    def metadata(self):
        if self._metadata == None:
            # We haven't previously built the metadata - go do that and cache for later
            metadata = {}
            for plugin in list(self.plugins.values()):
                # get the config and push it up with the metadata
                config = self.config.get(plugin.textkey, {})
                try:
                    meta = format_metadata(plugin.get_metadata(config))
                except:
                    self.log.exception('error getting metadata from %r', plugin.textkey)
                    continue
                self.log.info('got %d textkeys for %r', len(list(meta.keys())), plugin.textkey)
                label = getattr(plugin, 'label', plugin.textkey)
                metadata[plugin.textkey] = (label, meta)

            self._metadata = metadata

        return self._metadata

    # returns the metadata hash, used for comparing whether or not our metadata
    # has changed since we last pushed.  Uses local cache of values to avoid recomputing.
    def hashed_metadata(self):
        if self._hashed_metadata == None:
            flat_data = []

            for textkey, (label, plugin_metadata) in list(self.metadata.items()):
                plugin_options = []
                for resource_name, resource_metadata in list(plugin_metadata.items()):
                    resource_meta = sorted(repr(resource_metadata))
                    plugin_options.append(repr((resource_name, resource_metadata)))

                plugin_options.sort()
                flat_data.append(repr((textkey, plugin_options, label)))

            flat_data.sort()
            self._hashed_metadata = sha1(repr(flat_data).encode()).hexdigest()

        return self._hashed_metadata

    def is_metadata_stale(self):
        self.log.info('comparing metadata hashes %r with %r',
                      self.hashed_metadata(), self.db['last_metadata'])
        return self.db['last_metadata'] != self.hashed_metadata()


def format_metadata(metadata):
    """
    Go over the metadata entry and make sure the option resources
    are matching to the new style of dictionaries if they
    are a string.
    """
    for resource, entry in metadata.items():
        options = entry['options']
        new_options = []
        if not options:
            continue
        for option in options:
            if sys.version_info[0] == 3:
                if type(option) == str:  # TODO Verify that unicode is present on old python installs.
                    new_options.append({'resource': option})
            else:
                if type(option) in (str, unicode):  # TODO Verify that unicode is present on old python installs.
                    new_options.append({'resource': option})

        if new_options:
            entry['options'] = new_options
        if not entry.get('options_schema'):
            entry['options_schema'] = {'resource': 'string'}
    return metadata