moved of_g to c_gen.of_g_legacy, introduced loxi_globals
as of_g includes loads of legacy data (and a representation that
is increasingly tailored to the c backend only), I've moved it to
c_gen.of_g. In its place, I have created loxi_globals, which gives
access to the list of versions and the IR.
Also loxi reorg: move command line processing to cmdline.py
diff --git a/loxigen.py b/loxigen.py
index 7b78f63..ba62288 100755
--- a/loxigen.py
+++ b/loxigen.py
@@ -69,248 +69,27 @@
"""
-import sys
-
+from collections import OrderedDict, defaultdict
+import copy
+import glob
+from optparse import OptionParser
+import os
import re
import string
-import os
-import glob
-import copy
-import collections
-import of_g
-import loxi_front_end.type_maps as type_maps
+import sys
+
+import cmdline
+from loxi_globals import OFVersions
+import loxi_globals
import loxi_utils.loxi_utils as loxi_utils
-import loxi_front_end.c_parse_utils as c_parse_utils
-import loxi_front_end.identifiers as identifiers
import pyparsing
import loxi_front_end.parser as parser
-import loxi_front_end.translation as translation
import loxi_front_end.frontend as frontend
-from loxi_ir import *
+import loxi_ir
from generic_utils import *
root_dir = os.path.dirname(os.path.realpath(__file__))
-# TODO: Put these in a class so they get documented
-
-## Dict indexed by version giving all info related to version
-#
-# This is local; after processing, the information is stored in
-# of_g variables.
-versions = {}
-
-def config_sanity_check():
- """
- Check the configuration for basic consistency
-
- @fixme Needs update for generic language support
- """
-
- rv = True
- # For now, only "error" supported for get returns
- if config_check("copy_semantics") != "read":
- debug("Only 'read' is supported for copy_semantics");
- rv = False
- if config_check("get_returns") != "error":
- debug("Only 'error' is supported for get-accessor return types\m");
- rv = False
- if not config_check("use_fn_ptrs") and not config_check("gen_unified_fns"):
- debug("Must have gen_fn_ptrs and/or gen_unified_fns set in config")
- rv = False
- if config_check("use_obj_id"):
- debug("use_obj_id is set but not yet supported (change \
-config_sanity_check if it is)")
- rv = False
- if config_check("gen_unified_macros") and config_check("gen_unified_fns") \
- and config_check("gen_unified_macro_lower"):
- debug("Conflict: Cannot generate unified functions and lower case \
-unified macros")
- rv = False
-
- return rv
-
-def add_class(wire_version, cls, members):
- """
- Process a class for the given version and update the unified
- list of classes as needed.
-
- @param wire_version The wire version for this class defn
- @param cls The name of the class being added
- @param members The list of members with offsets calculated
- """
- memid = 0
-
- sig = loxi_utils.class_signature(members)
- if cls in of_g.unified:
- uc = of_g.unified[cls]
- if wire_version in uc:
- debug("Error adding %s to unified. Wire ver %d exists" %
- (cls, wire_version))
- sys.exit(1)
- uc[wire_version] = {}
- # Check for a matching signature
- for wver in uc:
- if type(wver) != type(0): continue
- if wver == wire_version: continue
- if not "use_version" in uc[wver]:
- if sig == loxi_utils.class_signature(uc[wver]["members"]):
- log("Matched %s, ver %d to ver %d" %
- (cls, wire_version, wver))
- # have a match with existing version
- uc[wire_version]["use_version"] = wver
- # What else to do?
- return
- else: # Haven't seen this entry before
- log("Adding %s to unified list, ver %d" % (cls, wire_version))
- of_g.unified[cls] = dict(union={})
- uc = of_g.unified[cls]
-
- # At this point, need to add members for this version
- uc[wire_version] = dict(members = members)
-
- # Per member processing:
- # Add to union list (I'm sure there's a better way)
- # Check if it's a list
- union = uc["union"]
- if not cls in of_g.ordered_members:
- of_g.ordered_members[cls] = []
- for member in members:
- m_name = member["name"]
- m_type = member["m_type"]
- if m_name.find("pad") == 0:
- continue
- if m_name in union:
- if not m_type == union[m_name]["m_type"]:
- debug("ERROR: CLASS: %s. VERSION %d. MEMBER: %s. TYPE: %s" %
- (cls, wire_version, m_name, m_type))
- debug(" Type conflict adding member to unified set.")
- debug(" Current union[%s]:" % m_name)
- debug(union[m_name])
- sys.exit(1)
- else:
- union[m_name] = dict(m_type=m_type, memid=memid)
- memid += 1
- if not m_name in of_g.ordered_members[cls]:
- of_g.ordered_members[cls].append(m_name)
-
-def update_offset(cls, wire_version, name, offset, m_type):
- """
- Update (and return) the offset based on type.
- @param cls The parent class
- @param wire_version The wire version being processed
- @param name The name of the data member
- @param offset The current offset
- @param m_type The type declaration being processed
- @returns A pair (next_offset, len_update) next_offset is the new offset
- of the next object or -1 if this is a var-length object. len_update
- is the increment that should be added to the length. Note that (for
- of_match_v3) it is variable length, but it adds 8 bytes to the fixed
- length of the object
- If offset is already -1, do not update
- Otherwise map to base type and count and update (if possible)
- """
- if offset < 0: # Don't update offset once set to -1
- return offset, 0
-
- count, base_type = c_parse_utils.type_dec_to_count_base(m_type)
-
- len_update = 0
- if base_type in of_g.of_mixed_types:
- base_type = of_g.of_mixed_types[base_type][wire_version]
-
- base_class = base_type[:-2]
- if (base_class, wire_version) in of_g.is_fixed_length:
- bytes = of_g.base_length[(base_class, wire_version)]
- else:
- if base_type == "of_match_v3_t":
- # This is a special case: it has non-zero min length
- # but is variable length
- bytes = -1
- len_update = 8
- elif base_type in of_g.of_base_types:
- bytes = of_g.of_base_types[base_type]["bytes"]
- else:
- print "UNKNOWN TYPE for %s %s: %s" % (cls, name, base_type)
- log("UNKNOWN TYPE for %s %s: %s" % (cls, name, base_type))
- bytes = -1
-
- # If bytes
- if bytes > 0:
- len_update = count * bytes
-
- if bytes == -1:
- return -1, len_update
-
- return offset + (count * bytes), len_update
-
-def calculate_offsets_and_lengths(ordered_classes, classes, wire_version):
- """
- Generate the offsets for fixed offset class members
- Also calculate the class_sizes when possible.
-
- @param classes The classes to process
- @param wire_version The wire version for this set of classes
-
- Updates global variables
- """
-
- lists = set()
-
- # Generate offsets
- for cls in ordered_classes:
- fixed_offset = 0 # The last "good" offset seen
- offset = 0
- last_offset = 0
- last_name = "-"
- for member in classes[cls]:
- m_type = member["m_type"]
- name = member["name"]
- if last_offset == -1:
- if name == "pad":
- log("Skipping pad for special offset for %s" % cls)
- else:
- log("SPECIAL OFS: Member %s (prev %s), class %s ver %d" %
- (name, last_name, cls, wire_version))
- if (((cls, name) in of_g.special_offsets) and
- (of_g.special_offsets[(cls, name)] != last_name)):
- debug("ERROR: special offset prev name changed")
- debug(" cls %s. name %s. version %d. was %s. now %s" %
- cls, name, wire_version,
- of_g.special_offsets[(cls, name)], last_name)
- sys.exit(1)
- of_g.special_offsets[(cls, name)] = last_name
-
- member["offset"] = offset
- if m_type.find("list(") == 0:
- (list_name, base_type) = loxi_utils.list_name_extract(m_type)
- lists.add(list_name)
- member["m_type"] = list_name + "_t"
- offset = -1
- elif m_type.find("struct") == 0:
- debug("ERROR found struct: %s.%s " % (cls, name))
- sys.exit(1)
- elif m_type == "octets":
- log("offset gen skipping octets: %s.%s " % (cls, name))
- offset = -1
- else:
- offset, len_update = update_offset(cls, wire_version, name,
- offset, m_type)
- if offset != -1:
- fixed_offset = offset
- else:
- fixed_offset += len_update
- log("offset is -1 for %s.%s version %d " %
- (cls, name, wire_version))
- last_offset = offset
- last_name = name
- of_g.base_length[(cls, wire_version)] = fixed_offset
- if (offset != -1):
- of_g.is_fixed_length.add((cls, wire_version))
- for list_type in lists:
- classes[list_type] = []
- of_g.ordered_classes[wire_version].append(list_type)
- of_g.base_length[(list_type, wire_version)] = 0
-
def process_input_file(filename):
"""
Process an input file
@@ -332,78 +111,13 @@
# Create the OFInput from the AST
try:
- ofinput = frontend.create_ofinput(ast)
+ ofinput = frontend.create_ofinput(filename, ast)
except frontend.InputError as e:
print "Error in %s: %s" % (os.path.basename(filename), str(e))
sys.exit(1)
return ofinput
-def order_and_assign_object_ids():
- """
- Order all classes and assign object ids to all classes.
-
- This is done to promote a reasonable order of the objects, putting
- messages first followed by non-messages. No assumptions should be
- made about the order, nor about contiguous numbering. However, the
- numbers should all be reasonably small allowing arrays indexed by
- these enum values to be defined.
- """
-
- # Generate separate message and non-message ordered lists
- for cls in of_g.unified:
- if loxi_utils.class_is_message(cls):
- of_g.ordered_messages.append(cls)
- elif loxi_utils.class_is_list(cls):
- of_g.ordered_list_objects.append(cls)
- else:
- of_g.ordered_non_messages.append(cls)
-
- of_g.ordered_messages.sort()
- of_g.ordered_pseudo_objects.sort()
- of_g.ordered_non_messages.sort()
- of_g.ordered_list_objects.sort()
- of_g.standard_class_order.extend(of_g.ordered_messages)
- of_g.standard_class_order.extend(of_g.ordered_non_messages)
- of_g.standard_class_order.extend(of_g.ordered_list_objects)
-
- # This includes pseudo classes for which most code is not generated
- of_g.all_class_order.extend(of_g.ordered_messages)
- of_g.all_class_order.extend(of_g.ordered_non_messages)
- of_g.all_class_order.extend(of_g.ordered_list_objects)
- of_g.all_class_order.extend(of_g.ordered_pseudo_objects)
-
- # Assign object IDs
- for cls in of_g.ordered_messages:
- of_g.unified[cls]["object_id"] = of_g.object_id
- of_g.object_id += 1
- for cls in of_g.ordered_non_messages:
- of_g.unified[cls]["object_id"] = of_g.object_id
- of_g.object_id += 1
- for cls in of_g.ordered_list_objects:
- of_g.unified[cls]["object_id"] = of_g.object_id
- of_g.object_id += 1
- for cls in of_g.ordered_pseudo_objects:
- of_g.unified[cls] = {}
- of_g.unified[cls]["object_id"] = of_g.object_id
- of_g.object_id += 1
-
-
-def initialize_versions():
- """
- Create an empty datastructure for each target version.
- """
-
- for wire_version in of_g.target_version_list:
- version_name = of_g.of_version_wire2name[wire_version]
- of_g.wire_ver_map[wire_version] = version_name
- versions[version_name] = dict(
- version_name = version_name,
- wire_version = wire_version,
- classes = {})
- of_g.ordered_classes[wire_version] = []
-
-
def read_input():
"""
Read in from files given on command line and update global state
@@ -411,7 +125,7 @@
@fixme Should select versions to support from command line
"""
- ofinputs_by_version = collections.defaultdict(lambda: [])
+ ofinputs_by_version = defaultdict(lambda: [])
filenames = sorted(glob.glob("%s/openflow_input/*" % root_dir))
# Ignore emacs backup files
@@ -421,245 +135,50 @@
log("Processing struct file: " + filename)
ofinput = process_input_file(filename)
- # Populate global state
for wire_version in ofinput.wire_versions:
ofinputs_by_version[wire_version].append(ofinput)
- version_name = of_g.of_version_wire2name[wire_version]
+ return ofinputs_by_version
- for ofclass in ofinput.classes:
- of_g.ordered_classes[wire_version].append(ofclass.name)
- legacy_members = []
- pad_count = 0
- for m in ofclass.members:
- if type(m) == OFPadMember:
- m_name = 'pad%d' % pad_count
- if m_name == 'pad0': m_name = 'pad'
- legacy_members.append(dict(m_type='uint8_t[%d]' % m.length,
- name=m_name))
- pad_count += 1
- else:
- # HACK the C backend does not yet support of_oxm_t
- if m.oftype == 'of_oxm_t':
- m_type = 'of_octets_t'
- else:
- enum = find(lambda e: e.name == m.oftype, ofinput.enums)
- if enum and "wire_type" in enum.params:
- m_type = enum.params["wire_type"]
- else:
- m_type = m.oftype
- legacy_members.append(dict(m_type=m_type, name=m.name))
- versions[version_name]['classes'][ofclass.name] = legacy_members
+def build_ir(ofinputs_by_version):
+ classes = []
+ enums = []
+ for wire_version, ofinputs in ofinputs_by_version.items():
+ version = OFVersions.from_wire(wire_version)
+ ofprotocol = loxi_ir.build_protocol(version, ofinputs)
+ loxi_globals.ir[version] = ofprotocol
- for enum in ofinput.enums:
- for entry in enum.entries:
- identifiers.add_identifier(
- translation.loxi_name(entry.name),
- entry.name, enum.name, entry.value, wire_version,
- of_g.identifiers, of_g.identifiers_by_group)
+ loxi_globals.unified = loxi_ir.build_unified_ir(loxi_globals.ir)
- for wire_version, ofinputs in ofinputs_by_version.items():
- ofprotocol = OFProtocol(wire_version=wire_version, classes=[], enums=[])
- for ofinput in ofinputs:
- ofprotocol.classes.extend(ofinput.classes)
- ofprotocol.enums.extend(ofinput.enums)
- ofprotocol.classes.sort(key=lambda ofclass: ofclass.name)
- of_g.ir[wire_version] = ofprotocol
-
-def populate_type_maps():
- """
- Use the type members in the IR to fill out the legacy type_maps.
- """
-
- def split_inherited_cls(cls):
- if cls == 'of_meter_band_stats': # HACK not a subtype of of_meter_band
- return None, None
- for parent in sorted(type_maps.inheritance_data.keys(), reverse=True):
- if cls.startswith(parent):
- return (parent, cls[len(parent)+1:])
- return None, None
-
- def find_experimenter(parent, cls):
- for experimenter in sorted(of_g.experimenter_name_to_id.keys(), reverse=True):
- prefix = parent + '_' + experimenter
- if cls.startswith(prefix) and cls != prefix:
- return experimenter
- return None
-
- def find_type_value(ofclass, m_name):
- for m in ofclass.members:
- if isinstance(m, OFTypeMember) and m.name == m_name:
- return m.value
- raise KeyError("ver=%d, cls=%s, m_name=%s" % (wire_version, cls, m_name))
-
- # Most inheritance classes: actions, instructions, etc
- for wire_version, protocol in of_g.ir.items():
- for ofclass in protocol.classes:
- cls = ofclass.name
- parent, subcls = split_inherited_cls(cls)
- if not (parent and subcls):
- continue
- if parent == 'of_oxm':
- type_len = find_type_value(ofclass, 'type_len')
- oxm_class = (type_len >> 16) & 0xffff
- if oxm_class != 0x8000:
- # Do not include experimenter OXMs in the main table
- val = type_maps.invalid_type
- else:
- val = (type_len >> 8) & 0xff
- else:
- val = find_type_value(ofclass, 'type')
- type_maps.inheritance_data[parent][wire_version][subcls] = val
-
- # Extensions (only actions for now)
- experimenter = find_experimenter(parent, cls)
- if parent == 'of_action' and experimenter:
- val = find_type_value(ofclass, 'subtype')
- type_maps.extension_action_subtype[wire_version][experimenter][cls] = val
- if wire_version >= of_g.VERSION_1_3:
- cls2 = parent + "_id" + cls[len(parent):]
- type_maps.extension_action_id_subtype[wire_version][experimenter][cls2] = val
-
- # Messages
- for wire_version, protocol in of_g.ir.items():
- for ofclass in protocol.classes:
- cls = ofclass.name
- # HACK (though this is what loxi_utils.class_is_message() does)
- if not [x for x in ofclass.members if isinstance(x, OFDataMember) and x.name == 'xid']:
- continue
- if type_maps.class_is_virtual(cls):
- continue
- subcls = cls[3:]
- val = find_type_value(ofclass, 'type')
- if not val in type_maps.message_types[wire_version].values():
- type_maps.message_types[wire_version][subcls] = val
-
- # Extensions
- experimenter = find_experimenter('of', cls)
- if experimenter:
- val = find_type_value(ofclass, 'subtype')
- type_maps.extension_message_subtype[wire_version][experimenter][cls] = val
-
- type_maps.generate_maps()
-
-def analyze_input():
- """
- Add information computed from the input, including offsets and
- lengths of struct members and the set of list and action_id types.
- """
-
- # Generate header classes for inheritance parents
- for wire_version, ordered_classes in of_g.ordered_classes.items():
- classes = versions[of_g.of_version_wire2name[wire_version]]['classes']
- for cls in ordered_classes:
- if cls in type_maps.inheritance_map:
- new_cls = cls + '_header'
- of_g.ordered_classes[wire_version].append(new_cls)
- classes[new_cls] = classes[cls]
-
- # Generate action_id classes for OF 1.3
- for wire_version, ordered_classes in of_g.ordered_classes.items():
- if not wire_version in [of_g.VERSION_1_3]:
- continue
- classes = versions[of_g.of_version_wire2name[wire_version]]['classes']
- for cls in ordered_classes:
- if not loxi_utils.class_is_action(cls):
- continue
- action = cls[10:]
- if action == '' or action == 'header':
- continue
- name = "of_action_id_" + action
- members = classes["of_action"][:]
- of_g.ordered_classes[wire_version].append(name)
- if type_maps.action_id_is_extension(name, wire_version):
- # Copy the base action classes thru subtype
- members = classes["of_action_" + action][:4]
- classes[name] = members
-
- # @fixme If we support extended actions in OF 1.3, need to add IDs
- # for them here
-
- for wire_version in of_g.wire_ver_map.keys():
- version_name = of_g.of_version_wire2name[wire_version]
- calculate_offsets_and_lengths(
- of_g.ordered_classes[wire_version],
- versions[version_name]['classes'],
- wire_version)
-
-def unify_input():
- """
- Create Unified View of Objects
- """
-
- global versions
-
- # Add classes to unified in wire-format order so that it is easier
- # to generate things later
- keys = versions.keys()
- keys.sort(reverse=True)
- for version in keys:
- wire_version = versions[version]["wire_version"]
- classes = versions[version]["classes"]
- for cls in of_g.ordered_classes[wire_version]:
- add_class(wire_version, cls, classes[cls])
-
-
-def log_all_class_info():
- """
- Log the results of processing the input
-
- Debug function
- """
-
- for cls in of_g.unified:
- for v in of_g.unified[cls]:
- if type(v) == type(0):
- log("cls: %s. ver: %d. base len %d. %s" %
- (str(cls), v, of_g.base_length[(cls, v)],
- loxi_utils.class_is_var_len(cls,v) and "not fixed"
- or "fixed"))
- if "use_version" in of_g.unified[cls][v]:
- log("cls %s: v %d mapped to %d" % (str(cls), v,
- of_g.unified[cls][v]["use_version"]))
- if "members" in of_g.unified[cls][v]:
- for member in of_g.unified[cls][v]["members"]:
- log(" %-20s: type %-20s. offset %3d" %
- (member["name"], member["m_type"],
- member["offset"]))
-
-def generate_all_files():
- """
- Create the files for the language target
- """
- lang_module.generate()
+################################################################
+#
+# Debug
+#
+################################################################
if __name__ == '__main__':
- of_g.loxigen_log_file = open("loxigen.log", "w")
- of_g.loxigen_dbg_file = sys.stdout
-
- of_g.process_commandline()
+ (options, args, target_versions) = cmdline.process_commandline()
# @fixme Use command line params to select log
- if not config_sanity_check():
+ logging.basicConfig(level = logging.INFO if not options.verbose else logging.DEBUG)
+
+ # Import the language file
+ lang_file = "lang_%s" % options.lang
+ lang_module = __import__(lang_file)
+
+ if hasattr(lang_module, "config_sanity_check") and not lang_module.config_sanity_check():
debug("Config sanity check failed\n")
sys.exit(1)
- # Import the language file
- lang_file = "lang_%s" % of_g.options.lang
- lang_module = __import__(lang_file)
-
# If list files, just list auto-gen files to stdout and exit
- if of_g.options.list_files:
+ if options.list_files:
for name in lang_module.targets:
- print of_g.options.install_dir + '/' + name
+ print options.install_dir + '/' + name
sys.exit(0)
- log("\nGenerating files for target language %s\n" % of_g.options.lang)
+ log("\nGenerating files for target language %s\n" % options.lang)
- initialize_versions()
- read_input()
- populate_type_maps()
- analyze_input()
- unify_input()
- order_and_assign_object_ids()
- log_all_class_info()
- generate_all_files()
+ loxi_globals.OFVersions.target_versions = target_versions
+ inputs = read_input()
+ build_ir(inputs)
+ #log_all_class_info()
+ lang_module.generate(options.install_dir)