loxigen.py

#!/usr/bin/python
# Copyright 2013, Big Switch Networks, Inc.
#
# LoxiGen is licensed under the Eclipse Public License, version 1.0 (EPL), with
# the following special exception:
#
# LOXI Exception
#
# As a special exception to the terms of the EPL, you may distribute libraries
# generated by LoxiGen (LoxiGen Libraries) under the terms of your choice, provided
# that copyright and licensing notices generated by LoxiGen are not altered or removed
# from the LoxiGen Libraries and the notice provided below is (i) included in
# the LoxiGen Libraries, if distributed in source code form and (ii) included in any
# documentation for the LoxiGen Libraries, if distributed in binary form.
#
# Notice: "Copyright 2013, Big Switch Networks, Inc. This library was generated by the LoxiGen Compiler."
#
# You may not use this file except in compliance with the EPL or LOXI Exception. You may obtain
# a copy of the EPL at:
#
# http://www.eclipse.org/legal/epl-v10.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# EPL for the specific language governing permissions and limitations
# under the EPL.

"""
@brief
Process openflow header files to create language specific LOXI interfaces

First cut at simple python script for processing input files

Internal notes

An input file for each supported OpenFlow version is passed in
on the command line.

Expected input file format:

These will probably be collapsed into a python dict or something

The first line has the ofC version identifier and nothing else
The second line has the openflow wire protocol value and nothing else

The main content is struct elements for each OF recognized class.
These are taken from current versions of openflow.h but are modified
a bit.  See Overview for more information.

Class canonical form:   A list of entries, each of which is a 
pair "type, name;".  The exception is when type is the keyword
'list' in which the syntax is "list(type) name;".

From this, internal representations are generated:  For each
version, a dict indexed by class name.  One element (members) is
an array giving the member name and type.  From this, wire offsets
can be calculated.


@fixme Clean up the lang module architecture.  It should provide a
list of files that it wants to generate and maps to the filenames,
subdirectory names and generation functions.  It should also be 
defined as a class, probably with the constructor taking the 
language target.

@fixme Clean up global data structures such as versions and of_g
structures.  They should probably be a class or classes as well.

"""

import sys

import re
import string
import os
import glob
import copy
import of_g
import loxi_front_end.oxm as oxm
import loxi_front_end.type_maps as type_maps
import loxi_utils.loxi_utils as loxi_utils
import loxi_front_end.of_h_utils as of_h_utils
import loxi_front_end.c_parse_utils as c_parse_utils
import loxi_front_end.identifiers as identifiers
import pyparsing
import loxi_front_end.parser as parser

from generic_utils import *

root_dir = os.path.dirname(os.path.realpath(__file__))

# TODO:  Put these in a class so they get documented

## Dict indexed by version giving all info related to version
#
# This is local; after processing, the information is stored in
# of_g variables.
versions = {}

def config_sanity_check():
    """
    Check the configuration for basic consistency

    @fixme Needs update for generic language support
    """

    rv = True
    # For now, only "error" supported for get returns
    if config_check("copy_semantics") != "read":
        debug("Only 'read' is supported for copy_semantics");
        rv = False        
    if config_check("get_returns") != "error":
        debug("Only 'error' is supported for get-accessor return types\m");
        rv = False        
    if not config_check("use_fn_ptrs") and not config_check("gen_unified_fns"):
        debug("Must have gen_fn_ptrs and/or gen_unified_fns set in config")
        rv = False
    if config_check("use_obj_id"):
        debug("use_obj_id is set but not yet supported (change \
config_sanity_check if it is)")
        rv = False
    if config_check("gen_unified_macros") and config_check("gen_unified_fns") \
            and config_check("gen_unified_macro_lower"):
        debug("Conflict: Cannot generate unified functions and lower case \
unified macros")
        rv = False
        
    return rv

def add_class(wire_version, cls, members):
    """
    Process a class for the given version and update the unified 
    list of classes as needed.

    @param wire_version The wire version for this class defn
    @param cls The name of the class being added
    @param members The list of members with offsets calculated
    """
    memid = 0

    sig = loxi_utils.class_signature(members)
    if cls in of_g.unified:
        uc = of_g.unified[cls]
        if wire_version in uc:
            debug("Error adding %s to unified. Wire ver %d exists" %
                  (cls, wire_version))
            sys.exit(1)
        uc[wire_version] = {}
        # Check for a matching signature
        for wver in uc:
            if type(wver) != type(0): continue
            if wver == wire_version: continue
            if not "use_version" in uc[wver]:
                if sig == loxi_utils.class_signature(uc[wver]["members"]):
                    log("Matched %s, ver %d to ver %d" % 
                          (cls, wire_version, wver))
                    # have a match with existing version
                    uc[wire_version]["use_version"] = wver
                    # What else to do?
                    return
    else:  # Haven't seen this entry before
        log("Adding %s to unified list, ver %d" % (cls, wire_version))
        of_g.unified[cls] = dict(union={})
        uc = of_g.unified[cls]

    # At this point, need to add members for this version
    uc[wire_version] = dict(members = members)

    # Per member processing:
    #  Add to union list (I'm sure there's a better way)
    #  Check if it's a list
    union = uc["union"]
    if not cls in of_g.ordered_members:
        of_g.ordered_members[cls] = []
    for member in members:
        m_name = member["name"]
        m_type = member["m_type"]
        if m_name.find("pad") == 0:
            continue
        if m_name in union:
            if not m_type == union[m_name]["m_type"]:
                debug("ERROR:   CLASS: %s. VERSION %d. MEMBER: %s. TYPE: %s" %
                      (cls, wire_version, m_name, m_type))
                debug("    Type conflict adding member to unified set.")
                debug("    Current union[%s]:" % m_name)
                debug(union[m_name])
                sys.exit(1)
        else:
            union[m_name] = dict(m_type=m_type, memid=memid)
            memid += 1
        if not m_name in of_g.ordered_members[cls]:
            of_g.ordered_members[cls].append(m_name)

def update_offset(cls, wire_version, name, offset, m_type):
    """
    Update (and return) the offset based on type.
    @param cls The parent class
    @param wire_version The wire version being processed
    @param name The name of the data member
    @param offset The current offset
    @param m_type The type declaration being processed
    @returns A pair (next_offset, len_update)  next_offset is the new offset
    of the next object or -1 if this is a var-length object.  len_update
    is the increment that should be added to the length.  Note that (for
    of_match_v3) it is variable length, but it adds 8 bytes to the fixed
    length of the object
    If offset is already -1, do not update
    Otherwise map to base type and count and update (if possible)
    """
    if offset < 0:    # Don't update offset once set to -1
        return offset, 0

    count, base_type = c_parse_utils.type_dec_to_count_base(m_type)

    len_update = 0
    if base_type in of_g.of_mixed_types:
        base_type = of_g.of_mixed_types[base_type][wire_version]

    base_class = base_type[:-2]
    if (base_class, wire_version) in of_g.is_fixed_length:
        bytes = of_g.base_length[(base_class, wire_version)]
    else:
        if base_type == "of_match_v3_t":
            # This is a special case: it has non-zero min length
            # but is variable length
            bytes = -1
            len_update = 8
        elif base_type in of_g.of_base_types:
            bytes = of_g.of_base_types[base_type]["bytes"]
        else:
            print "UNKNOWN TYPE for %s %s: %s" % (cls, name, base_type)
            log("UNKNOWN TYPE for %s %s: %s" % (cls, name, base_type))
            bytes = -1

    # If bytes
    if bytes > 0:
        len_update = count * bytes

    if bytes == -1:
        return -1, len_update

    return offset + (count * bytes), len_update

def calculate_offsets_and_lengths(ordered_classes, classes, wire_version):
    """
    Generate the offsets for fixed offset class members
    Also calculate the class_sizes when possible.

    @param classes The classes to process
    @param wire_version The wire version for this set of classes

    Updates global variables
    """

    lists = set()

    # Generate offsets
    for cls in ordered_classes:
        fixed_offset = 0 # The last "good" offset seen
        offset = 0
        last_offset = 0
        last_name = "-"
        for member in classes[cls]:
            m_type = member["m_type"]
            name = member["name"]
            if last_offset == -1:
                if name == "pad":
                    log("Skipping pad for special offset for %s" % cls)
                else:
                    log("SPECIAL OFS: Member %s (prev %s), class %s ver %d" % 
                          (name, last_name, cls, wire_version))
                    if (((cls, name) in of_g.special_offsets) and
                        (of_g.special_offsets[(cls, name)] != last_name)):
                        debug("ERROR: special offset prev name changed")
                        debug("  cls %s. name %s. version %d. was %s. now %s" %
                              cls, name, wire_version, 
                              of_g.special_offsets[(cls, name)], last_name)
                        sys.exit(1)
                    of_g.special_offsets[(cls, name)] = last_name

            member["offset"] = offset
            if m_type.find("list(") == 0:
                (list_name, base_type) = loxi_utils.list_name_extract(m_type)
                lists.add(list_name)
                member["m_type"] = list_name + "_t"
                offset = -1
            elif m_type.find("struct") == 0:
                debug("ERROR found struct: %s.%s " % (cls, name))
                sys.exit(1)
            elif m_type == "octets":
                log("offset gen skipping octets: %s.%s " % (cls, name))
                offset = -1
            else:
                offset, len_update = update_offset(cls, wire_version, name, 
                                                  offset, m_type)
                if offset != -1:
                    fixed_offset = offset
                else:
                    fixed_offset += len_update
                    log("offset is -1 for %s.%s version %d " % 
                        (cls, name, wire_version))
            last_offset = offset
            last_name = name
        of_g.base_length[(cls, wire_version)] = fixed_offset
        if (offset != -1):
            of_g.is_fixed_length.add((cls, wire_version))
    for list_type in lists:
        classes[list_type] = []
        of_g.ordered_classes[wire_version].append(list_type)
        of_g.base_length[(list_type, wire_version)] = 0

def process_input_file(filename):
    """
    Process an input file

    @param filename The input filename

    @returns (wire_version, classes), where wire_version is the integer wire
    protocol number and classes is the dict of all classes processed from the
    file.

    @todo Add support for parsing enums like we do structs
    """

    # Parse the input file
    try:
        ast = parser.parse(open(filename, 'r').read())
    except pyparsing.ParseBaseException as e:
        print "Parse error in %s: %s" % (os.path.basename(filename), str(e))
        sys.exit(1)

    ofinput = of_g.OFInput()

    # Now for each structure, generate lists for each member
    for s in ast:
        if s[0] == 'struct':
            name = s[1].replace("ofp_", "of_", 1)
            members = [dict(m_type=x[0], name=x[1]) for x in s[2]]
            ofinput.classes[name] = members
            ofinput.ordered_classes.append(name)
            if name in type_maps.inheritance_map:
                # Clone class into header class and add to list
                ofinput.classes[name + "_header"] = members[:]
                ofinput.ordered_classes.append(name + "_header")
        elif s[0] == 'metadata':
            if s[1] == 'version':
                log("Found version: wire version " + s[2])
                if s[2] == 'any':
                    ofinput.wire_versions.update(of_g.wire_ver_map.keys())
                elif int(s[2]) in of_g.supported_wire_protos:
                    ofinput.wire_versions.add(int(s[2]))
                else:
                    debug("Unrecognized wire protocol version")
                    sys.exit(1)
                found_wire_version = True

    if not ofinput.wire_versions:
        debug("Missing #version metadata")
        sys.exit(1)

    return ofinput

def order_and_assign_object_ids():
    """
    Order all classes and assign object ids to all classes.

    This is done to promote a reasonable order of the objects, putting
    messages first followed by non-messages.  No assumptions should be
    made about the order, nor about contiguous numbering.  However, the
    numbers should all be reasonably small allowing arrays indexed by 
    these enum values to be defined.
    """

    # Generate separate message and non-message ordered lists
    for cls in of_g.unified:
        if loxi_utils.class_is_message(cls):
            of_g.ordered_messages.append(cls)
        elif loxi_utils.class_is_list(cls):
            of_g.ordered_list_objects.append(cls)
        else:
            of_g.ordered_non_messages.append(cls)

    of_g.ordered_pseudo_objects.append("of_stats_request")
    of_g.ordered_pseudo_objects.append("of_stats_reply")
    of_g.ordered_pseudo_objects.append("of_flow_mod")

    of_g.ordered_messages.sort()
    of_g.ordered_pseudo_objects.sort()
    of_g.ordered_non_messages.sort()
    of_g.ordered_list_objects.sort()
    of_g.standard_class_order.extend(of_g.ordered_messages)
    of_g.standard_class_order.extend(of_g.ordered_non_messages)
    of_g.standard_class_order.extend(of_g.ordered_list_objects)

    # This includes pseudo classes for which most code is not generated
    of_g.all_class_order.extend(of_g.ordered_messages)
    of_g.all_class_order.extend(of_g.ordered_non_messages)
    of_g.all_class_order.extend(of_g.ordered_list_objects)
    of_g.all_class_order.extend(of_g.ordered_pseudo_objects)

    # Assign object IDs
    for cls in of_g.ordered_messages:
        of_g.unified[cls]["object_id"] = of_g.object_id
        of_g.object_id += 1
    for cls in of_g.ordered_non_messages:
        of_g.unified[cls]["object_id"] = of_g.object_id
        of_g.object_id += 1
    for cls in of_g.ordered_list_objects:
        of_g.unified[cls]["object_id"] = of_g.object_id
        of_g.object_id += 1
    for cls in of_g.ordered_pseudo_objects:
        of_g.unified[cls] = {}
        of_g.unified[cls]["object_id"] = of_g.object_id
        of_g.object_id += 1

def process_canonical_file(version, filename):
    """
    Read in contents of openflow.h file filename and process it
    @param version The wire version number
    @param filename The name of the openflow header file for this version

    Updates of_g.identifiers dictionary.  See of_g.py
    """
    log("Processing canonical file %s, version %d" % (filename, version))
    f = open(filename, 'r')
    all_lines = f.readlines()
    contents = " ".join(all_lines)
    identifiers.add_identifiers(of_g.identifiers, of_g.identifiers_by_group,
                                version, contents)


def initialize_versions():
    """
    Create an empty datastructure for each target version.
    """

    for wire_version in of_g.target_version_list:
        version_name = of_g.of_version_wire2name[wire_version]
        of_g.wire_ver_map[wire_version] = version_name
        versions[version_name] = dict(
            version_name = version_name,
            wire_version = wire_version,
            classes = {})
        of_g.ordered_classes[wire_version] = []


def read_input():
    """
    Read in from files given on command line and update global state

    @fixme Should select versions to support from command line
    """

    filenames = sorted(glob.glob("%s/openflow_input/*" % root_dir))

    for filename in filenames:
        log("Processing struct file: " + filename)
        ofinput = process_input_file(filename)

        # Populate global state
        for wire_version in ofinput.wire_versions:
            version_name = of_g.of_version_wire2name[wire_version]
            versions[version_name]['classes'].update(copy.deepcopy(ofinput.classes))
            of_g.ordered_classes[wire_version].extend(ofinput.ordered_classes)

def add_extra_classes():
    """
    Add classes that are generated by Python code instead of from the
    input files.
    """

    for wire_version in [of_g.VERSION_1_2, of_g.VERSION_1_3]:
        version_name = of_g.of_version_wire2name[wire_version]
        oxm.add_oxm_classes_1_2(versions[version_name]['classes'], wire_version)

def analyze_input():
    """
    Add information computed from the input, including offsets and
    lengths of struct members and the set of list and action_id types.
    """

    # Generate action_id classes for OF 1.3
    for wire_version, ordered_classes in of_g.ordered_classes.items():
        if not wire_version in [of_g.VERSION_1_3]:
            continue
        classes = versions[of_g.of_version_wire2name[wire_version]]['classes']
        for cls in ordered_classes:
            if not loxi_utils.class_is_action(cls):
                continue
            action = cls[10:]
            if action == '' or action == 'header':
                continue
            name = "of_action_id_" + action
            members = classes["of_action"][:]
            of_g.ordered_classes[wire_version].append(name)
            if type_maps.action_id_is_extension(name, wire_version):
                # Copy the base action classes thru subtype
                members = classes["of_action_" + action][:4]
            classes[name] = members

    # @fixme If we support extended actions in OF 1.3, need to add IDs
    # for them here

    for wire_version in of_g.wire_ver_map.keys():
        version_name = of_g.of_version_wire2name[wire_version]
        calculate_offsets_and_lengths(
            of_g.ordered_classes[wire_version],
            versions[version_name]['classes'],
            wire_version)

def unify_input():
    """
    Create Unified View of Objects
    """

    global versions

    # Add classes to unified in wire-format order so that it is easier 
    # to generate things later
    keys = versions.keys()
    keys.sort(reverse=True)
    for version in keys:
        wire_version = versions[version]["wire_version"]
        classes = versions[version]["classes"]
        for cls in of_g.ordered_classes[wire_version]:
            add_class(wire_version, cls, classes[cls])


def log_all_class_info():
    """
    Log the results of processing the input

    Debug function
    """

    for cls in of_g.unified:
        for v in of_g.unified[cls]:
            if type(v) == type(0):
                log("cls: %s. ver: %d. base len %d. %s" %
                    (str(cls), v, of_g.base_length[(cls, v)],
                     loxi_utils.class_is_var_len(cls,v) and "not fixed"
                     or "fixed"))
                if "use_version" in of_g.unified[cls][v]:
                    log("cls %s: v %d mapped to %d" % (str(cls), v, 
                           of_g.unified[cls][v]["use_version"]))
                if "members" in of_g.unified[cls][v]:
                    for member in of_g.unified[cls][v]["members"]:
                        log("   %-20s: type %-20s. offset %3d" %
                            (member["name"], member["m_type"],
                             member["offset"]))

def generate_all_files():
    """
    Create the files for the language target
    """
    for (name, fn) in lang_module.targets.items():
        path = of_g.options.install_dir + '/' + name
        os.system("mkdir -p %s" % os.path.dirname(path))
        with open(path, "w") as outfile:
            fn(outfile, os.path.basename(name))
        print("Wrote contents for " + name)

if __name__ == '__main__':
    of_g.loxigen_log_file = open("loxigen.log", "w")
    of_g.loxigen_dbg_file = sys.stdout

    of_g.process_commandline()
    # @fixme Use command line params to select log

    if not config_sanity_check():
        debug("Config sanity check failed\n")
        sys.exit(1)

    # Import the language file
    lang_file = "lang_%s" % of_g.options.lang
    lang_module = __import__(lang_file)

    # If list files, just list auto-gen files to stdout and exit
    if of_g.options.list_files:
        for name in lang_module.targets:
            print of_g.options.install_dir + '/' + name
        sys.exit(0)

    log("\nGenerating files for target language %s\n" % of_g.options.lang)

    # Generate identifier code
    for version in of_g.target_version_list:
        version_tag = of_g.param_version_names[version]
        filename = "%s/canonical/openflow.h-%s" % (root_dir, version_tag)
        process_canonical_file(version, filename)

    initialize_versions()
    read_input()
    add_extra_classes()
    analyze_input()
    unify_input()
    order_and_assign_object_ids()
    log_all_class_info()
    generate_all_files()