Integrating YANG live compilation into YANG runtime.
- Bumped ONOS dependency on ONOS YANG tools 2.2.0-b4.
- Added CLI to compile YANG models.
- Added GUI capability to compile YANG models via drag-n-drop or file upload.
- Fixed defect in propagating self-contained JAR apps through the cluster.
Change-Id: Icbd2a588bf1ffe0282e12d3d10a117e0957c3084
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java
new file mode 100644
index 0000000..3061da6
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.yang.impl;
+
+import com.google.common.io.ByteStreams;
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onosproject.app.ApplicationAdminService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.yang.YangLiveCompilerService;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+/**
+ * Compiles the provided YANG source files and installs the resulting model extension.
+ */
+@Command(scope = "onos", name = "compile-model",
+ description = "Compiles the provided YANG source files and installs the resulting model extension")
+public class YangCompileCommand extends AbstractShellCommand {
+
+ @Option(name = "-c", aliases = "--compile-only",
+ description = "Only compile, but do not install the model")
+ private boolean compileOnly = false;
+
+ @Option(name = "-f", aliases = "--force",
+ description = "Force reinstall if already installed")
+ private boolean forceReinstall = false;
+
+ @Argument(index = 0, name = "modelId",
+ description = "Model ID", required = true)
+ String modelId = null;
+
+ @Argument(index = 1, name = "url",
+ description = "URL to the YANG source file(s); .yang, .zip or .jar file",
+ required = true)
+ String url = null;
+
+ @Override
+ protected void execute() {
+ try {
+ InputStream yangJar = new URL(url).openStream();
+ YangLiveCompilerService compiler = get(YangLiveCompilerService.class);
+ if (compileOnly) {
+ ByteStreams.copy(compiler.compileYangFiles(modelId, yangJar), System.out);
+ } else {
+ ApplicationAdminService appService = get(ApplicationAdminService.class);
+
+ if (forceReinstall) {
+ ApplicationId appId = appService.getId(modelId);
+ if (appId != null && appService.getApplication(appId) != null) {
+ appService.uninstall(appId);
+ }
+ }
+
+ appService.install(compiler.compileYangFiles(modelId, yangJar));
+ appService.activate(appService.getId(modelId));
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java
new file mode 100644
index 0000000..d109dc3
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.yang.impl;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Service;
+import org.onosproject.yang.YangLiveCompilerService;
+import org.onosproject.yang.compiler.tool.DefaultYangCompilationParam;
+import org.onosproject.yang.compiler.tool.YangCompilerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static com.google.common.io.ByteStreams.toByteArray;
+import static com.google.common.io.Files.createParentDirs;
+import static com.google.common.io.Files.write;
+import static java.nio.file.Files.walkFileTree;
+
+/**
+ * Represents implementation of YANG live compiler manager.
+ */
+@Service
+@Component(immediate = true)
+public class YangLiveCompilerManager implements YangLiveCompilerService {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private static final String ZIP_MAGIC = "PK";
+
+ @Activate
+ public void activate() {
+ log.info("Started");
+ }
+
+ @Deactivate
+ public void deactivate() {
+ log.info("Stopped");
+ }
+
+ @Override
+ public InputStream compileYangFiles(String modelId,
+ InputStream yangSources) throws IOException {
+ // Generate temporary directory where the work will happen.
+ File root = Files.createTempDir();
+ log.info("Compiling YANG model to {}", root);
+
+ // Unpack the input stream
+ File yangRoot = unpackYangSources(root, yangSources);
+
+ // Run the YANG compilation phase
+ File javaRoot = runYangCompiler(root, yangRoot, modelId);
+
+ // Run the Java compilation phase
+ File classRoot = runJavaCompiler(root, javaRoot, modelId);
+
+ // Run the JAR assembly phase
+ File jarFile = runJarAssembly(root, classRoot, modelId);
+
+ // Return the final JAR file as input stream
+ return new FileInputStream(jarFile);
+ }
+
+ // Unpacks the given input stream into the YANG root subdirectory of the specified root directory.
+ private File unpackYangSources(File root, InputStream yangSources) throws IOException {
+ File yangRoot = new File(root, "yang/");
+ if (yangRoot.mkdirs()) {
+ // Unpack the yang sources into the newly created directory
+ byte[] cache = toByteArray(yangSources);
+ InputStream bis = new ByteArrayInputStream(cache);
+ if (isZipArchive(cache)) {
+ extractZipArchive(yangRoot, bis);
+ } else {
+ extractYangFile(yangRoot, bis);
+ }
+ return yangRoot;
+ }
+ throw new IOException("Unable to create yang source root");
+ }
+
+ // Extracts the YANG source stream into the specified directory.
+ private void extractYangFile(File dir, InputStream stream) throws IOException {
+ ByteStreams.copy(stream, new FileOutputStream(new File(dir, "model.yang")));
+ }
+
+ // Extracts the ZIP stream into the specified directory.
+ private void extractZipArchive(File dir, InputStream stream) throws IOException {
+ ZipInputStream zis = new ZipInputStream(stream);
+ ZipEntry entry;
+ while ((entry = zis.getNextEntry()) != null) {
+ if (!entry.isDirectory()) {
+ byte[] data = toByteArray(zis);
+ zis.closeEntry();
+ File file = new File(dir, entry.getName());
+ createParentDirs(file);
+ write(data, file);
+ }
+ }
+ zis.close();
+ }
+
+ // Runs the YANG compiler on the YANG sources in the specified directory.
+ private File runYangCompiler(File root, File yangRoot, String modelId) throws IOException {
+ File javaRoot = new File(root, "java/");
+ if (javaRoot.mkdirs()) {
+ // Prepare the compilation parameter
+ DefaultYangCompilationParam.Builder param = DefaultYangCompilationParam.builder()
+ .setCodeGenDir(new File(javaRoot, "src").toPath())
+ .setMetadataGenDir(new File(javaRoot, "schema").toPath())
+ .setModelId(modelId);
+
+ // TODO: How to convey YANG dependencies? "/dependencies" directory?
+
+ // Iterate over all files and add all YANG sources.
+ walkFileTree(Paths.get(yangRoot.getAbsolutePath()),
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
+ throws IOException {
+ if (attributes.isRegularFile() && file.toString().endsWith(".yang")) {
+ param.addYangFile(file);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+
+ // Run the YANG compiler and collect the results
+ new YangCompilerManager().compileYangFiles(param.build());
+ return javaRoot;
+ }
+ throw new IOException("Unable to create Java results root");
+ }
+
+ // Runs the Java compilation on the Java sources generated by YANG compiler.
+ private File runJavaCompiler(File root, File javaRoot, String modelId) throws IOException {
+ File classRoot = new File(root, "classes/");
+ if (classRoot.mkdirs()) {
+ File compilerScript = writeResource("onos-yang-javac", root);
+ writeResource("YangModelRegistrator.java", root);
+ execute(new String[]{
+ "bash",
+ compilerScript.getAbsolutePath(),
+ javaRoot.getAbsolutePath(),
+ classRoot.getAbsolutePath(),
+ modelId
+ });
+ return classRoot;
+ }
+ throw new IOException("Unable to create class results root");
+ }
+
+ // Run the JAR assembly on the classes root and include any YANG sources as well.
+ private File runJarAssembly(File root, File classRoot, String modelId) throws IOException {
+ File jarFile = new File(root, "model.jar");
+ File jarScript = writeResource("onos-yang-jar", root);
+ writeResource("app.xml", root);
+ writeResource("features.xml", root);
+ writeResource("YangModelRegistrator.xml", root);
+ execute(new String[]{
+ "bash",
+ jarScript.getAbsolutePath(),
+ classRoot.getAbsolutePath(),
+ jarFile.getAbsolutePath(),
+ modelId
+ });
+ return jarFile;
+ }
+
+ // Writes the specified resource as a file in the given directory.
+ private File writeResource(String resourceName, File dir) throws IOException {
+ File script = new File(dir, resourceName);
+ write(toByteArray(getClass().getResourceAsStream("/" + resourceName)), script);
+ return script;
+ }
+
+ // Indicates whether the stream encoded in the given bytes is a ZIP archive.
+ private boolean isZipArchive(byte[] bytes) {
+ return substring(bytes, ZIP_MAGIC.length()).equals(ZIP_MAGIC);
+ }
+
+ // Returns the substring of maximum possible length from the specified bytes.
+ private String substring(byte[] bytes, int length) {
+ return new String(bytes, 0, Math.min(bytes.length, length), StandardCharsets.UTF_8);
+ }
+
+ // Executes the given command arguments as a system command.
+ private void execute(String[] command) throws IOException {
+ try {
+ Process process = Runtime.getRuntime().exec(command);
+ byte[] output = toByteArray(process.getInputStream());
+ byte[] error = toByteArray(process.getErrorStream());
+ int code = process.waitFor();
+ if (code != 0) {
+ log.info("Command failed: status={}, output={}, error={}",
+ code, new String(output), new String(error));
+ }
+ } catch (InterruptedException e) {
+ log.error("Interrupted executing command {}", command, e);
+ }
+ }
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java
new file mode 100644
index 0000000..69fef62
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.yang.impl;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.yang.model.YangModel;
+import org.onosproject.yang.model.YangModule;
+import org.onosproject.yang.runtime.YangModelRegistry;
+
+/**
+ * Lists registered YANG models.
+ */
+@Command(scope = "onos", name = "models",
+ description = "Lists registered YANG models")
+public class YangModelsListCommand extends AbstractShellCommand {
+
+
+ private static final String FORMAT = "modelId=%s moduleName=%s moduleRevision=%s";
+ private static final String MODULE_NAME = "moduleName";
+ private static final String MODULE_REVISION = "moduleRevision";
+
+ @Override
+ protected void execute() {
+ YangModelRegistry service = get(YangModelRegistry.class);
+
+ if (outputJson()) {
+ print("%s", json(service));
+ } else {
+ for (YangModel model : service.getModels()) {
+ for (YangModule module : model.getYangModules()) {
+ print(FORMAT, model.getYangModelId(),
+ module.getYangModuleId().moduleName(),
+ module.getYangModuleId().revision());
+ }
+ }
+ }
+ }
+
+ // Produces JSON structure.
+ private JsonNode json(YangModelRegistry service) {
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode result = mapper.createObjectNode();
+ for (YangModel model : service.getModels()) {
+ ArrayNode modelNode = mapper.createArrayNode();
+ result.set(model.getYangModelId(), modelNode);
+ for (YangModule module : model.getYangModules()) {
+ ObjectNode moduleNode = mapper.createObjectNode();
+ modelNode.add(moduleNode);
+ moduleNode.put(MODULE_NAME, module.getYangModuleId().moduleName());
+ moduleNode.put(MODULE_REVISION, module.getYangModuleId().revision());
+ }
+ }
+ return result;
+ }
+
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java
new file mode 100644
index 0000000..5d9a076
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java
@@ -0,0 +1,196 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.yang.impl;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onosproject.core.CoreService;
+import org.onosproject.yang.YangClassLoaderRegistry;
+import org.onosproject.yang.model.ModelConverter;
+import org.onosproject.yang.model.ModelObjectData;
+import org.onosproject.yang.model.NodeKey;
+import org.onosproject.yang.model.ResourceData;
+import org.onosproject.yang.model.ResourceId;
+import org.onosproject.yang.model.RpcContext;
+import org.onosproject.yang.model.SchemaContext;
+import org.onosproject.yang.model.SchemaContextProvider;
+import org.onosproject.yang.model.YangModel;
+import org.onosproject.yang.model.YangModule;
+import org.onosproject.yang.model.YangModuleId;
+import org.onosproject.yang.runtime.CompositeData;
+import org.onosproject.yang.runtime.CompositeStream;
+import org.onosproject.yang.runtime.ModelRegistrationParam;
+import org.onosproject.yang.runtime.RuntimeContext;
+import org.onosproject.yang.runtime.YangModelRegistry;
+import org.onosproject.yang.runtime.YangRuntimeService;
+import org.onosproject.yang.runtime.YangSerializer;
+import org.onosproject.yang.runtime.YangSerializerRegistry;
+import org.onosproject.yang.runtime.impl.DefaultModelConverter;
+import org.onosproject.yang.runtime.impl.DefaultSchemaContextProvider;
+import org.onosproject.yang.runtime.impl.DefaultYangModelRegistry;
+import org.onosproject.yang.runtime.impl.DefaultYangRuntimeHandler;
+import org.onosproject.yang.runtime.impl.DefaultYangSerializerRegistry;
+import org.onosproject.yang.serializers.json.JsonSerializer;
+import org.onosproject.yang.serializers.xml.XmlSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Represents implementation of YANG runtime manager.
+ */
+@Service
+@Component(immediate = true)
+public class YangRuntimeManager implements YangModelRegistry,
+ YangSerializerRegistry, YangRuntimeService, ModelConverter,
+ SchemaContextProvider, YangClassLoaderRegistry {
+
+ private static final String APP_ID = "org.onosproject.yang";
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+ protected CoreService coreService;
+
+ private DefaultYangModelRegistry modelRegistry;
+ private DefaultYangSerializerRegistry serializerRegistry;
+ private DefaultYangRuntimeHandler runtimeService;
+ private DefaultModelConverter modelConverter;
+ private DefaultSchemaContextProvider schemaContextProvider;
+
+ private Map<String, ClassLoader> classLoaders;
+
+ @Activate
+ public void activate() {
+ coreService.registerApplication(APP_ID);
+ serializerRegistry = new DefaultYangSerializerRegistry();
+ modelRegistry = new DefaultYangModelRegistry();
+ runtimeService = new DefaultYangRuntimeHandler(serializerRegistry, modelRegistry);
+ schemaContextProvider = new DefaultSchemaContextProvider(modelRegistry);
+ serializerRegistry.registerSerializer(new JsonSerializer());
+ serializerRegistry.registerSerializer(new XmlSerializer());
+ modelConverter = new DefaultModelConverter(modelRegistry);
+ classLoaders = new ConcurrentHashMap<>();
+ log.info("Started");
+ }
+
+ @Deactivate
+ public void deactivate() {
+ log.info("Stopped");
+ }
+
+
+ @Override
+ public void registerModel(ModelRegistrationParam p) {
+ modelRegistry.registerModel(p);
+ }
+
+ @Override
+ public void unregisterModel(ModelRegistrationParam p) {
+ modelRegistry.unregisterModel(p);
+ }
+
+ @Override
+ public Set<YangModel> getModels() {
+ return modelRegistry.getModels();
+ }
+
+ @Override
+ public YangModel getModel(String s) {
+ return modelRegistry.getModel(s);
+ }
+
+ @Override
+ public YangModule getModule(YangModuleId yangModuleId) {
+ return modelRegistry.getModule(yangModuleId);
+ }
+
+ @Override
+ public void registerSerializer(YangSerializer ys) {
+ serializerRegistry.registerSerializer(ys);
+ }
+
+ @Override
+ public void unregisterSerializer(YangSerializer ys) {
+ serializerRegistry.unregisterSerializer(ys);
+ }
+
+ @Override
+ public Set<YangSerializer> getSerializers() {
+ return serializerRegistry.getSerializers();
+ }
+
+ @Override
+ public CompositeData decode(CompositeStream cs, RuntimeContext rc) {
+ return runtimeService.decode(cs, rc);
+ }
+
+ @Override
+ public CompositeStream encode(CompositeData cd, RuntimeContext rc) {
+ return runtimeService.encode(cd, rc);
+ }
+
+ @Override
+ public ModelObjectData createModel(ResourceData resourceData) {
+ return modelConverter.createModel(resourceData);
+ }
+
+ @Override
+ public ResourceData createDataNode(ModelObjectData modelObjectData) {
+ return modelConverter.createDataNode(modelObjectData);
+ }
+
+ @Override
+ public SchemaContext getSchemaContext(ResourceId resourceId) {
+ checkNotNull(resourceId, " resource id can't be null.");
+ NodeKey key = resourceId.nodeKeys().get(0);
+ if (resourceId.nodeKeys().size() == 1 &&
+ "/".equals(key.schemaId().name())) {
+ return modelRegistry;
+ }
+ log.info("To be implemented.");
+ return null;
+ }
+
+ @Override
+ public RpcContext getRpcContext(ResourceId resourceId) {
+ return schemaContextProvider.getRpcContext(resourceId);
+ }
+
+ @Override
+ public ClassLoader getClassLoader(String modelId) {
+ return classLoaders.get(modelId);
+ }
+
+ @Override
+ public void registerClassLoader(String modelId, ClassLoader classLoader) {
+ classLoaders.put(modelId, classLoader);
+ }
+
+ @Override
+ public void unregisterClassLoader(String modelId) {
+ classLoaders.remove(modelId);
+ }
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java b/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java
new file mode 100755
index 0000000..5140248
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementations of various YANG runtime services.
+ */
+package org.onosproject.yang.impl;
\ No newline at end of file