main.py

#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER
#
# Copyright (c) 2018 Juniper Networks, Inc.
# All rights reserved.
#
# Use is subject to license terms.
#
# Licensed under the Apache License, Version 2.0 (the ?License?); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at http://www.apache.org/licenses/LICENSE-2.0.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import cherrypy
import datetime
import hashlib
import logging
import os
import yaml
import re

from jinja2 import Environment, FileSystemLoader
from jnpr.junos.utils.config import Config
from jnpr.junos import Device
from jnpr.junos.exception import ConfigLoadError, CommitError, ConnectRefusedError, ConnectTimeoutError
from data.fr import FlowRoutesTable, FlowFilterTable

# set up logging for the main program
logFormatter = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(format=logFormatter, level=logging.INFO)
logger = logging.getLogger(__name__)

# suppressing the INFO logging from ncclient module
logging.getLogger('ncclient').setLevel(logging.WARNING)
# suppressing the INFO logging from cherrypy module
logging.getLogger('cherrypy').setLevel(logging.WARNING)

class MyDev(object):

    def __init__(self):

        self.dev_user = None
        self.dev_pw = None
        self.age_out_interval = None
        self.flow_active = dict()
        self.flow_config = dict()
        self.filter_active = dict()
        self.routers = list()
        self.vrfs = list()

    def addNewFlowRoute(self, flowRouteData=None):

        if flowRouteData['vrf'] == 'default':
            flowRouteData.pop('vrf', None)

        # env = Environment(autoescape=False,
        #                   loader=FileSystemLoader('./template'), trim_blocks=False, lstrip_blocks=False)
        # template = env.get_template('set-flow-route.conf')
        # print template.render(flowRouteData)

        my_router = None
        for router in self.routers:
            for name, value in router.items():
                if 'rr' in value['type']:
                    my_router = [value['ip']]

        try:
            with Device(host=my_router[0], user=self.dev_user, password=self.dev_pw) as dev:

                cu = Config(dev)
                cu.lock()

                cu.load(template_path='template/set-flow-route.conf', template_vars=flowRouteData)
                cu.commit()
                cu.unlock()
                logger.info('Flow Route added ' + str(flowRouteData))

        except ConfigLoadError as cle:
            logger.warning(cle.message)
            return False, cle.message
        except (ConnectRefusedError, ConnectTimeoutError) as cre:
            logger.warning('NETCONF connection error: {}'.format(cre))
            return False

        # self.flow_config[flowRouteData['flowRouteName']] = {
        #     'dstPrefix': flowRouteData['dstPrefix'] if 'dstPrefix' in flowRouteData else None,
        #     'srcPrefix': flowRouteData['srcPrefix'] if 'srcPrefix' in flowRouteData else None,
        #     'protocol': flowRouteData['protocol'] if 'protocol' in flowRouteData else None,
        #     'dstPort': flowRouteData['dstPort'] if 'dstPort' in flowRouteData else None,
        #     'srcPort': flowRouteData['srcPort'] if 'srcPort' in flowRouteData else None,
        #     'action': flowRouteData['action']}
        return True, 'Successfully added new flow route'

    def modFlowRoute(self, flowRouteData=None):

        if flowRouteData['vrf'] == 'default':
            flowRouteData.pop('vrf', None)

        # env = Environment(autoescape=False,
        #                   loader=FileSystemLoader('./template'), trim_blocks=False, lstrip_blocks=False)
        # template = env.get_template('set-flow-route.conf')
        # print(template.render(flowRouteData))

        my_router = None
        for router in self.routers:

            for name, value in router.items():
                if 'rr' in value['type']:
                    my_router = [value['ip']]

        try:
            with Device(host=my_router[0], user=self.dev_user, password=self.dev_pw) as dev:

                cu = Config(dev)
                cu.lock()
                cu.load(template_path='template/set-flow-route.conf', template_vars=flowRouteData)
                cu.commit()
                cu.unlock()
                logger.info('Flow Route modified ' + str(flowRouteData))

        except CommitError as ce:
            logger.warning(ce.message)
            return False, ce.message
        except (ConnectRefusedError, ConnectTimeoutError) as cre:
            logger.warning('NETCONF connection error: {}'.format(cre))
            return False

        # self.flow_config[flowRouteData['flowRouteName']] = {
        #     'dstPrefix': flowRouteData['dstPrefix'] if 'dstPrefix' in flowRouteData else None,
        #     'srcPrefix': flowRouteData['srcPrefix'] if 'srcPrefix' in flowRouteData else None,
        #     'protocol': flowRouteData['protocol'] if 'protocol' in flowRouteData else None,
        #     'dstPort': flowRouteData['dstPort'] if 'dstPort' in flowRouteData else None,
        #     'srcPort': flowRouteData['srcPort'] if 'srcPort' in flowRouteData else None,
        #     'action': flowRouteData['action']}

        return True, 'Successfully modified flow route'

    def delFlowRoute(self, flowRouteData=None):

        my_router = None
        for router in self.routers:

            for name, value in router.items():
                if 'rr' in value['type']:
                    my_router = [value['ip']]
                    
        if 'default' in flowRouteData['vrf']:
            flowRouteData.pop('vrf', None)
        
        flowRouteData.update({'flowRouteName': flowRouteData['name']})

        try:
            with Device(host=my_router[0], user=self.dev_user, password=self.dev_pw) as dev:

                cu = Config(dev)
                cu.lock()
                cu.load(template_path='template/delete-flow-route.conf', template_vars=flowRouteData, merge=True)
                cu.commit()
                cu.unlock()
                logger.info('Flow Route deleted ' + str(flowRouteData))

        except ConfigLoadError as cle:
            logger.warning(cle.message)
            return False, cle.message
        except (ConnectRefusedError, ConnectTimeoutError) as cre:
            logger.warning('NETCONF connection error: {}'.format(cre))
            return False

        self.flow_config.pop(flowRouteData['flowRouteName'], None)

        return True, 'Sucessfully deleted flow route'

    def getActiveFlowRoutes(self):

        t = datetime.datetime.strptime(self.age_out_interval, "%H:%M:%S")
        self.flow_active = dict()

        for router in self.routers:

            for name, value in router.items():

                try:
                    with Device(host=value['ip'], user=self.dev_user, password=self.dev_pw) as dev:

                        # data = dev.rpc.get_config(filter_xml='routing-options/flow/route/name')
                        frt = FlowRoutesTable(dev)
                        flowtables = list()
                        for vrf in self.vrfs:
                            if vrf == 'default':
                                flowtables.extend(['inetflow.0', 'inet6flow.0'])
                            else:
                                flowtables.extend(['{}.inetflow.0'.format(vrf), '{}.inet6flow.0'.format(vrf)])
                        # for flowtable in ['inetflow.0', 'inet6flow.0', 'trcps.inetflow.0', 'trcps.inet6flow.0']:
                        for flowtable in flowtables:
                            frt.get(table=flowtable)

                            _flowtable = flowtable.split('.')
                            vrf = ''
                            family = ''
                            _vrf = ['','']
                            if len(_flowtable) == 2:
                                vrf = 'default'
                                if _flowtable[0] == 'inetflow':
                                    family = 'v4'
                                if _flowtable[0] == 'inet6flow':
                                    family = 'v6'
                            if len(_flowtable) > 2:
                                vrf = _flowtable[0]
                                if _flowtable[1] == 'inetflow':
                                    family = 'v4'
                                if _flowtable[1] == 'inet6flow':
                                    family = 'v6'

                            for flow in frt:

                                destination = self.parseFlow(flow.destination)

                                # for index, item in enumerate(destination):
                                #     _item = item.split('=')
                                #     destination[index] = _item[1] if len(_item) > 1 else _item[0]

                                hash_object = hashlib.sha512('{0}{1}'.format(str(destination), str(value['ip'])).encode('utf-8'))
                                hex_dig = hash_object.hexdigest()
                                _age = dict()

                                if len(flow.age) <= 2:
                                    _age['current'] = datetime.timedelta(seconds=int(flow.age))
                                elif len(flow.age) == 4 or len(flow.age) == 5:
                                    ms = flow.age.split(':')
                                    _age['current'] = datetime.timedelta(minutes=int(ms[0]), seconds=int(ms[1]))
                                elif len(flow.age) == 7 or len(flow.age) == 8:
                                    ms = flow.age.split(':')
                                    _age['current'] = datetime.timedelta(hours=int(ms[0]), minutes=int(ms[1]),
                                                                        seconds=int(ms[2]))
                                else:
                                    pattern = r'(.*w)?(.*)\s(.*?):(.*?):(.*)'
                                    regex = re.compile(pattern)
                                    age = re.findall(regex, flow.age)
                                    if age[0][0][:-1] is '':
                                        weeks = 0
                                    else:
                                        weeks = int(age[0][0][:-1])
                                    _age['current'] = datetime.timedelta(
                                                                        weeks=weeks,
                                                                        days=int(age[0][1][:-1]), 
                                                                        hours=int(age[0][2]),
                                                                        minutes=int(age[0][3]), 
                                                                        seconds=int(age[0][4]))

                                pattern = r'([^\s]+)'
                                regex = re.compile(pattern)
                                _krt_actions = re.findall(regex, flow.tsi)

                                if len(_krt_actions) <= 4:
                                    krt_actions = _krt_actions
                                else:
                                    krt_actions = _krt_actions[4]

                                # Junos 14.1RX different XPATH for BGP communities
                                version = dev.facts['version'].split('R')[0].split('.')

                                if int(version[0]) <= 14 and int(version[1]) <= 1:

                                    if isinstance(flow.action_141, str):
                                        if 'traffic-action' in flow.action_141:
                                            commAction = flow.action_141.split(":")[1].lstrip().strip()
                                        else:
                                            commAction = flow.action_141

                                    elif isinstance(flow.action_141, list):
                                        commAction = flow.action_141[1].split(':')[1].lstrip().strip()
                                    else:
                                        commAction = flow.action_141

                                else:

                                    if isinstance(flow.action, str):
                                        if 'traffic-action' in flow.action:
                                            commAction = flow.action.split(":")[1].lstrip().strip()
                                        else:
                                            commAction = flow.action

                                    elif isinstance(flow.action, list):
                                        commAction = flow.action[1].split(':')[1].lstrip().strip()
                                    else:
                                        commAction = flow.action

                                if hex_dig not in self.flow_active:

                                    self.flow_active[hex_dig] = {'router': name, 'vrf': vrf, 'family': family, 'term': flow.term, 'destination': destination,
                                                                'commAction': commAction, 'krtAction': krt_actions,
                                                                'age': str(_age['current']),
                                                                'hash': hex_dig, 'status': 'new'}
                                else:

                                    if 'term:N/A' in flow['term']:
                                        self.flow_active.pop(hex_dig, None)

                                    if _age['current']:

                                        if _age['current'] > datetime.timedelta(hours=t.hour, minutes=t.minute,
                                                                                seconds=t.second):
                                            self.flow_active[hex_dig]['status'] = 'old'

                                    try:
                                        if hex_dig in self.flow_active:
                                            self.flow_active[hex_dig].update({'term': flow.term, 'destination': destination,
                                                                            'commAction': commAction,
                                                                            'krtAction': krt_actions,
                                                                            'age': str(_age['current'])})

                                    except KeyError as ke:
                                        return False, ke

                except (ConnectRefusedError, ConnectTimeoutError) as cre:
                    logger.warning('NETCONF connection error: {}'.format(cre))
                    return False

        return True, self.flow_active

    def getActiveFlowRouteFilter(self):

        if self.routers:

            for router in self.routers:

                for name, value in router.items():
                    self.filter_active[name] = list()

                    try:
                        with Device(host=value['ip'], user=self.dev_user, password=self.dev_pw) as dev:

                            frft = FlowFilterTable(dev)
                            flowfilters = list()
                            for vrf in self.vrfs:
                                flowfilters.extend(['__flowspec_{}_inet__'.format(vrf), '__flowspec_{}_inet6__'.format(vrf)])

                            for table in flowfilters:
                            # for table in ['__flowspec_default_inet__', '__flowspec_default_inet6__', '__flowspec_trcps_inet__', '__flowspec_trcps_inet6__']:

                                frft.get(filtername=table)

                                for filter in frft:

                                    data = self.parseFlow(filter.name)

                                    # for didx, item in enumerate(data):
                                    #     _item = item.split('=')
                                    #     data[didx] = _item[1] if len(_item) > 1 else _item[0]

                                    _vrf = table.split('_')
                                    family = ''
                                    if _vrf[4] == 'inet':
                                        family = 'v4'
                                    if _vrf[4] == 'inet6':
                                        family = 'v6'
                                    vrf = '{}'.format(_vrf[3])
                                    
                                    self.filter_active[name].append({'vrf': vrf, 'family': family, 'data': data, 'packet_count': filter.packet_count,
                                                                    'byte_count': filter.byte_count})

                    except (ConnectRefusedError, ConnectTimeoutError) as cre:
                        logger.warning('NETCONF connection error: {}'.format(cre))
                        return False

            return True, self.filter_active

    def loadFlowRouteConfig(self):

        dev_ip = list()

        for router in self.routers:

            for name, value in router.items():

                if 'rr' in value['type']:
                    dev_ip.append(value['ip'])

        try:
            with Device(host=dev_ip[0], user=self.dev_user, password=self.dev_pw, normalize=True) as dev:

                version = dev.facts['version'].split('R')[0].split('.')

                # Junos 14.1RX does not support json so let's go with XML here

                if int(version[0]) <= 14 and int(version[1]) <= 1:

                    # Retrieving all routing-options so we can get both v4 and v6.  Might also work for VRFs
                    data = dev.rpc.get_config(options={'format': 'xml'}, filter_xml='routing-options')
                    # Retrieving all routing-instances.  Should figure out how to limit to specified VRFs
                    data_trcps = dev.rpc.get_config(options={'format': 'xml'}, filter_xml='routing-instances')
                    
                    data.append(data_trcps)

                    # iterate only over <flow> children. Appears to enter into multiple levels implicitly
                    for routes in data.iter('flow'):
                        for route in routes.iter('route'):
                            my_list = list()

                            for item in route:

                                if 'name' in item.tag:
                                    my_list.append(item.text)
                                    self.flow_config[item.text] = {}

                                elif 'match' in item.tag:
                                    tag = None

                                    for child in item.iterchildren():

                                        if 'destination-port' in child.tag:
                                            tag = 'dstPort'
                                        elif 'source-port' in child.tag:
                                            tag = 'srcPort'
                                        elif 'destination' in child.tag:
                                            tag = 'dstPrefix'
                                        elif 'source' in child.tag:
                                            tag = 'srcPrefix'
                                        elif 'protocol' in child.tag:
                                            tag = 'protocol'
                                        
                                        # v6 flow routes have a <prefix> child inside src and dst items
                                        # this grabs the value and puts it in child.txt
                                        if tag in ['srcPrefix', 'dstPrefix']:
                                            for grandchild in child.iterchildren():
                                                if 'prefix' in grandchild.tag:
                                                    child.text = grandchild.text
                                                    
                                        self.flow_config[my_list[0]][tag] = child.text

                                elif 'then' in item.tag:

                                    _action = dict()

                                    for child in item.iterchildren():

                                        for value in child.iter():
                                            _action[child.tag] = {'value': value.text}

                                        self.flow_config[my_list[0]]['action'] = _action

                    return True, self.flow_config

                else:
                    for vrf in self.vrfs:
                        _vrf_data = dict()
                        _data = dict()
                        family = ''
                        if vrf == 'default':
                            try:
                                options = dev.rpc.get_config(options={'format': 'json'}, filter_xml='routing-options')
                            except IndexError as error:
                                logger.warn(error)
                            except (ConnectRefusedError, ConnectTimeoutError) as cre:
                                logger.warning('NETCONF connection error: {}'.format(cre))
                                return False
                            _vrf_data = options['configuration']['routing-options']
                        else:
                            try:
                                instances = dev.rpc.get_config(options={'format': 'json'}, filter_xml='routing-instances')
                                for instance in instances['configuration']['routing-instances']['instance']:
                                    if instance['name'] == vrf:
                                        _vrf_data = instance['routing-options']
                            except TypeError as error:
                                logger.warn('Routing instance {} not defined. Caught "{}"'.format(vrf, error))
                            except IndexError as error:
                                logger.warn(error)
                            except (ConnectRefusedError, ConnectTimeoutError) as cre:
                                logger.warning('NETCONF connection error: {}'.format(cre))
                                return False

                        for table in ['inet', 'inet6']:
                            if table == 'inet6':
                                if 'rib' in _vrf_data:
                                    if 'flow' in _vrf_data['rib'][0]:
                                        _data = _vrf_data['rib'][0]['flow']
                                        family = 'v6'
                            else:
                                if 'flow' in _vrf_data:
                                    _data = _vrf_data['flow']
                                    family = 'v4'
                            
                            if 'route' in _data:
                                for route in _data['route']:
                                    _action = dict()

                                    for key, value in route['then'].items():

                                        if value[0]:
                                            _action[key] = {'value': value}
                                        else:
                                            _action[key] = {'value': None}

                                    self.flow_config[route['name']] = {
                                        'vrf': vrf,
                                        'family': family,
                                        'protocol': route['match']['protocol'] if 'protocol' in route['match'] else None,
                                        'dstPort': route['match']['destination-port'] if 'destination-port' in route[
                                            'match'] else None,
                                        'srcPort': route['match']['source-port'] if 'source-port' in route['match'] else None,
                                        'action': _action
                                        }
                                    if family == 'v6':
                                        self.flow_config[route['name']].update({
                                            'dstPrefix': route['match']['destination']['prefix'] if 'destination' in route['match'] else None,
                                            'srcPrefix': route['match']['source']['prefix'] if 'source' in route['match'] else None
                                            })
                                    else:
                                        self.flow_config[route['name']].update({
                                            'dstPrefix': route['match']['destination'] if 'destination' in route['match'] else None,
                                            'srcPrefix': route['match']['source'] if 'source' in route['match'] else None
                                            })

                    return True, self.flow_config, self.vrfs

        except (ConnectRefusedError, ConnectTimeoutError) as cre:
            logger.warning('NETCONF connection error: {}'.format(cre))
            return False

    def parseFlow(self, flow=None):
        _flow = flow.split(',')
        data = dict()

        data['dst'] = _flow[0]
        data['src'] = _flow[1]
        key = ''

        for counter,item in enumerate(_flow):
          if counter == 0 or counter == 1:
            continue
          _item = re.split('=|:', item)
          if _item[0] is not '':
            key = _item[0]
            data[key] = [_item[1]]
          else:
            data[key].append(_item[1])

        return data

    def load_settings(self):

        with open('ui/config.yml', 'r') as fp:
            _config = fp.read()
            config = yaml.safe_load(_config)
            self.dev_user = config['dev_user']
            self.dev_pw = config['dev_pw']
            self.age_out_interval = config['age_out_interval']
            self.routers = config['routers']
            self.vrfs = config['vrfs']


class BGPFlow(object):

    @cherrypy.expose
    def index(self):
        return open('ui/index.html', 'r')


@cherrypy.expose
class BGPFlowWS(object):

    def __init__(self, my_dev=None):
        self.my_dev = my_dev

    @cherrypy.tools.json_out()
    @cherrypy.tools.json_in()
    def GET(self, action=None):

        if action == 'active':
            data = self.my_dev.getActiveFlowRoutes()

            return data

    @cherrypy.tools.json_out()
    @cherrypy.tools.json_in()
    def POST(self, action=None):

        if action == 'add':

            input_json = cherrypy.request.json
            resp = self.my_dev.addNewFlowRoute(flowRouteData=input_json)
            return resp

        elif action == 'mod':
            input_json = cherrypy.request.json
            resp = self.my_dev.modFlowRoute(flowRouteData=input_json)
            return resp

        elif action == 'del':
            input_json = cherrypy.request.json
            resp = self.my_dev.delFlowRoute(flowRouteData=input_json)
            return resp

        else:
            return False, 'Action not defined'


@cherrypy.expose
class Frt(object):

    def __init__(self, my_dev=None):
        self.my_dev = my_dev

    @cherrypy.tools.json_out()
    def POST(self):
        resp = self.my_dev.getActiveFlowRoutes()
        return resp


@cherrypy.expose
class Frtc(object):

    def __init__(self, my_dev=None):
        self.my_dev = my_dev

    @cherrypy.tools.json_out()
    def POST(self):
        resp = self.my_dev.loadFlowRouteConfig()
        return resp


@cherrypy.expose
class Frft(object):

    def __init__(self, my_dev=None):
        self.my_dev = my_dev

    @cherrypy.tools.json_out()
    def POST(self):
        resp = self.my_dev.getActiveFlowRouteFilter()
        return resp


if __name__ == '__main__':
    cherrypy.config.update({'log.screen': False,
                            'log.access_file': '',
                            'log.error_file': ''})
    conf = {
        '/': {
            'tools.sessions.on': True,
            'tools.staticdir.root': os.path.abspath(os.getcwd()),
            'tools.staticdir.on': True,
            'tools.staticdir.dir': 'ui'
        },
        '/api': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },
        '/api/frt': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },
        '/api/frct': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },
        '/api/frft': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
            'tools.response_headers.headers': [('Content-Type', 'text/plain')],
        },
    }

    my_dev = MyDev()
    my_dev.load_settings()
    webapp = BGPFlow()
    webapp.api = BGPFlowWS(my_dev=my_dev)
    webapp.api.frt = Frt(my_dev=my_dev)
    webapp.api.frct = Frtc(my_dev=my_dev)
    webapp.api.frft = Frft(my_dev=my_dev)
    cherrypy.config.update({'log.screen': False,
                            'server.socket_host': '0.0.0.0',
                            'server.socket_port': 8080,
                            })
    # suppressing the logging from cherrypy module if log.screen == False   
    cherrypy.log.error_log.propagate = True
    cherrypy.log.access_log.propagate = False
    cherrypy.quickstart(webapp, '/', conf)