Integrating YANG live compilation into YANG runtime.
- Bumped ONOS dependency on ONOS YANG tools 2.2.0-b4.
- Added CLI to compile YANG models.
- Added GUI capability to compile YANG models via drag-n-drop or file upload.
- Fixed defect in propagating self-contained JAR apps through the cluster.
Change-Id: Icbd2a588bf1ffe0282e12d3d10a117e0957c3084
diff --git a/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerManager.java b/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerManager.java
deleted file mode 100644
index a4fc857..0000000
--- a/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2017-present Open Networking Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.onosproject.yang;
-
-import com.google.common.annotations.Beta;
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
-import org.onosproject.core.CoreService;
-import org.onosproject.yang.compiler.api.YangCompilationParam;
-import org.onosproject.yang.compiler.api.YangCompiledOutput;
-import org.onosproject.yang.compiler.api.YangCompilerException;
-import org.onosproject.yang.compiler.api.YangCompilerService;
-import org.onosproject.yang.compiler.tool.YangCompilerManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Represents implementation of YANG live compiler manager.
- */
-@Beta
-@Service
-@Component(immediate = true)
-public class YangLiveCompilerManager implements YangCompilerService {
-
- private static final String APP_ID = "org.onosproject.yang";
- private final Logger log = LoggerFactory.getLogger(getClass());
- @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
- protected CoreService coreService;
-
- @Activate
- public void activate() {
- coreService.registerApplication(APP_ID);
- log.info("Started");
- }
-
- @Deactivate
- public void deactivate() {
- log.info("Stopped");
- }
-
- @Override
- public YangCompiledOutput compileYangFiles(YangCompilationParam param)
- throws IOException, YangCompilerException {
- return new YangCompilerManager().compileYangFiles(param);
- }
-}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerService.java b/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerService.java
new file mode 100644
index 0000000..de151d1
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/YangLiveCompilerService.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.yang;
+
+import com.google.common.annotations.Beta;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Runtime service for compiling YANG source files and packaging the compiled
+ * artifacts into a ready-to-load application.
+ */
+@Beta
+public interface YangLiveCompilerService {
+
+ /**
+ * Compiles the YANG source file(s) contained in the specified input
+ * stream either as individual YANG source file, or as a JAR/ZIP collection
+ * of YANG source files.
+ *
+ * @param modelId model identifier
+ * @param yangSources input stream containing a single YANG source file
+ * or a JAR/ZIP archive containing collection of YANG
+ * source files
+ * @return input stream containing a packaged ONOS application JAR file
+ * @throws IOException if issues arise when reading the yang sources stream
+ */
+ InputStream compileYangFiles(String modelId, InputStream yangSources) throws IOException;
+
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java
new file mode 100644
index 0000000..3061da6
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangCompileCommand.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.yang.impl;
+
+import com.google.common.io.ByteStreams;
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.apache.karaf.shell.commands.Option;
+import org.onosproject.app.ApplicationAdminService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.yang.YangLiveCompilerService;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+
+/**
+ * Compiles the provided YANG source files and installs the resulting model extension.
+ */
+@Command(scope = "onos", name = "compile-model",
+ description = "Compiles the provided YANG source files and installs the resulting model extension")
+public class YangCompileCommand extends AbstractShellCommand {
+
+ @Option(name = "-c", aliases = "--compile-only",
+ description = "Only compile, but do not install the model")
+ private boolean compileOnly = false;
+
+ @Option(name = "-f", aliases = "--force",
+ description = "Force reinstall if already installed")
+ private boolean forceReinstall = false;
+
+ @Argument(index = 0, name = "modelId",
+ description = "Model ID", required = true)
+ String modelId = null;
+
+ @Argument(index = 1, name = "url",
+ description = "URL to the YANG source file(s); .yang, .zip or .jar file",
+ required = true)
+ String url = null;
+
+ @Override
+ protected void execute() {
+ try {
+ InputStream yangJar = new URL(url).openStream();
+ YangLiveCompilerService compiler = get(YangLiveCompilerService.class);
+ if (compileOnly) {
+ ByteStreams.copy(compiler.compileYangFiles(modelId, yangJar), System.out);
+ } else {
+ ApplicationAdminService appService = get(ApplicationAdminService.class);
+
+ if (forceReinstall) {
+ ApplicationId appId = appService.getId(modelId);
+ if (appId != null && appService.getApplication(appId) != null) {
+ appService.uninstall(appId);
+ }
+ }
+
+ appService.install(compiler.compileYangFiles(modelId, yangJar));
+ appService.activate(appService.getId(modelId));
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java
new file mode 100644
index 0000000..d109dc3
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangLiveCompilerManager.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.onosproject.yang.impl;
+
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Service;
+import org.onosproject.yang.YangLiveCompilerService;
+import org.onosproject.yang.compiler.tool.DefaultYangCompilationParam;
+import org.onosproject.yang.compiler.tool.YangCompilerManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static com.google.common.io.ByteStreams.toByteArray;
+import static com.google.common.io.Files.createParentDirs;
+import static com.google.common.io.Files.write;
+import static java.nio.file.Files.walkFileTree;
+
+/**
+ * Represents implementation of YANG live compiler manager.
+ */
+@Service
+@Component(immediate = true)
+public class YangLiveCompilerManager implements YangLiveCompilerService {
+
+ private final Logger log = LoggerFactory.getLogger(getClass());
+
+ private static final String ZIP_MAGIC = "PK";
+
+ @Activate
+ public void activate() {
+ log.info("Started");
+ }
+
+ @Deactivate
+ public void deactivate() {
+ log.info("Stopped");
+ }
+
+ @Override
+ public InputStream compileYangFiles(String modelId,
+ InputStream yangSources) throws IOException {
+ // Generate temporary directory where the work will happen.
+ File root = Files.createTempDir();
+ log.info("Compiling YANG model to {}", root);
+
+ // Unpack the input stream
+ File yangRoot = unpackYangSources(root, yangSources);
+
+ // Run the YANG compilation phase
+ File javaRoot = runYangCompiler(root, yangRoot, modelId);
+
+ // Run the Java compilation phase
+ File classRoot = runJavaCompiler(root, javaRoot, modelId);
+
+ // Run the JAR assembly phase
+ File jarFile = runJarAssembly(root, classRoot, modelId);
+
+ // Return the final JAR file as input stream
+ return new FileInputStream(jarFile);
+ }
+
+ // Unpacks the given input stream into the YANG root subdirectory of the specified root directory.
+ private File unpackYangSources(File root, InputStream yangSources) throws IOException {
+ File yangRoot = new File(root, "yang/");
+ if (yangRoot.mkdirs()) {
+ // Unpack the yang sources into the newly created directory
+ byte[] cache = toByteArray(yangSources);
+ InputStream bis = new ByteArrayInputStream(cache);
+ if (isZipArchive(cache)) {
+ extractZipArchive(yangRoot, bis);
+ } else {
+ extractYangFile(yangRoot, bis);
+ }
+ return yangRoot;
+ }
+ throw new IOException("Unable to create yang source root");
+ }
+
+ // Extracts the YANG source stream into the specified directory.
+ private void extractYangFile(File dir, InputStream stream) throws IOException {
+ ByteStreams.copy(stream, new FileOutputStream(new File(dir, "model.yang")));
+ }
+
+ // Extracts the ZIP stream into the specified directory.
+ private void extractZipArchive(File dir, InputStream stream) throws IOException {
+ ZipInputStream zis = new ZipInputStream(stream);
+ ZipEntry entry;
+ while ((entry = zis.getNextEntry()) != null) {
+ if (!entry.isDirectory()) {
+ byte[] data = toByteArray(zis);
+ zis.closeEntry();
+ File file = new File(dir, entry.getName());
+ createParentDirs(file);
+ write(data, file);
+ }
+ }
+ zis.close();
+ }
+
+ // Runs the YANG compiler on the YANG sources in the specified directory.
+ private File runYangCompiler(File root, File yangRoot, String modelId) throws IOException {
+ File javaRoot = new File(root, "java/");
+ if (javaRoot.mkdirs()) {
+ // Prepare the compilation parameter
+ DefaultYangCompilationParam.Builder param = DefaultYangCompilationParam.builder()
+ .setCodeGenDir(new File(javaRoot, "src").toPath())
+ .setMetadataGenDir(new File(javaRoot, "schema").toPath())
+ .setModelId(modelId);
+
+ // TODO: How to convey YANG dependencies? "/dependencies" directory?
+
+ // Iterate over all files and add all YANG sources.
+ walkFileTree(Paths.get(yangRoot.getAbsolutePath()),
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
+ throws IOException {
+ if (attributes.isRegularFile() && file.toString().endsWith(".yang")) {
+ param.addYangFile(file);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+
+ // Run the YANG compiler and collect the results
+ new YangCompilerManager().compileYangFiles(param.build());
+ return javaRoot;
+ }
+ throw new IOException("Unable to create Java results root");
+ }
+
+ // Runs the Java compilation on the Java sources generated by YANG compiler.
+ private File runJavaCompiler(File root, File javaRoot, String modelId) throws IOException {
+ File classRoot = new File(root, "classes/");
+ if (classRoot.mkdirs()) {
+ File compilerScript = writeResource("onos-yang-javac", root);
+ writeResource("YangModelRegistrator.java", root);
+ execute(new String[]{
+ "bash",
+ compilerScript.getAbsolutePath(),
+ javaRoot.getAbsolutePath(),
+ classRoot.getAbsolutePath(),
+ modelId
+ });
+ return classRoot;
+ }
+ throw new IOException("Unable to create class results root");
+ }
+
+ // Run the JAR assembly on the classes root and include any YANG sources as well.
+ private File runJarAssembly(File root, File classRoot, String modelId) throws IOException {
+ File jarFile = new File(root, "model.jar");
+ File jarScript = writeResource("onos-yang-jar", root);
+ writeResource("app.xml", root);
+ writeResource("features.xml", root);
+ writeResource("YangModelRegistrator.xml", root);
+ execute(new String[]{
+ "bash",
+ jarScript.getAbsolutePath(),
+ classRoot.getAbsolutePath(),
+ jarFile.getAbsolutePath(),
+ modelId
+ });
+ return jarFile;
+ }
+
+ // Writes the specified resource as a file in the given directory.
+ private File writeResource(String resourceName, File dir) throws IOException {
+ File script = new File(dir, resourceName);
+ write(toByteArray(getClass().getResourceAsStream("/" + resourceName)), script);
+ return script;
+ }
+
+ // Indicates whether the stream encoded in the given bytes is a ZIP archive.
+ private boolean isZipArchive(byte[] bytes) {
+ return substring(bytes, ZIP_MAGIC.length()).equals(ZIP_MAGIC);
+ }
+
+ // Returns the substring of maximum possible length from the specified bytes.
+ private String substring(byte[] bytes, int length) {
+ return new String(bytes, 0, Math.min(bytes.length, length), StandardCharsets.UTF_8);
+ }
+
+ // Executes the given command arguments as a system command.
+ private void execute(String[] command) throws IOException {
+ try {
+ Process process = Runtime.getRuntime().exec(command);
+ byte[] output = toByteArray(process.getInputStream());
+ byte[] error = toByteArray(process.getErrorStream());
+ int code = process.waitFor();
+ if (code != 0) {
+ log.info("Command failed: status={}, output={}, error={}",
+ code, new String(output), new String(error));
+ }
+ } catch (InterruptedException e) {
+ log.error("Interrupted executing command {}", command, e);
+ }
+ }
+}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/YangModelsListCommand.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java
similarity index 79%
rename from apps/yang/src/main/java/org/onosproject/yang/YangModelsListCommand.java
rename to apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java
index 5915e88..69fef62 100644
--- a/apps/yang/src/main/java/org/onosproject/yang/YangModelsListCommand.java
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangModelsListCommand.java
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.onosproject.yang;
+package org.onosproject.yang.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -29,11 +29,11 @@
* Lists registered YANG models.
*/
@Command(scope = "onos", name = "models",
- description = "Lists registered YANG models")
+ description = "Lists registered YANG models")
public class YangModelsListCommand extends AbstractShellCommand {
- private static final String FORMAT = "moduleName=%s moduleRevision=%s modelId=%s";
+ private static final String FORMAT = "modelId=%s moduleName=%s moduleRevision=%s";
private static final String MODULE_NAME = "moduleName";
private static final String MODULE_REVISION = "moduleRevision";
@@ -44,11 +44,11 @@
if (outputJson()) {
print("%s", json(service));
} else {
- for (YangModel model: service.getModels()) {
+ for (YangModel model : service.getModels()) {
for (YangModule module : model.getYangModules()) {
- print(FORMAT, module.getYangModuleId().moduleName(),
- module.getYangModuleId().revision(),
- modelId(model));
+ print(FORMAT, model.getYangModelId(),
+ module.getYangModuleId().moduleName(),
+ module.getYangModuleId().revision());
}
}
}
@@ -58,9 +58,9 @@
private JsonNode json(YangModelRegistry service) {
ObjectMapper mapper = new ObjectMapper();
ObjectNode result = mapper.createObjectNode();
- for (YangModel model: service.getModels()) {
+ for (YangModel model : service.getModels()) {
ArrayNode modelNode = mapper.createArrayNode();
- result.set(modelId(model), modelNode);
+ result.set(model.getYangModelId(), modelNode);
for (YangModule module : model.getYangModules()) {
ObjectNode moduleNode = mapper.createObjectNode();
modelNode.add(moduleNode);
@@ -71,7 +71,4 @@
return result;
}
- private String modelId(YangModel m) {
- return "YM" + Math.abs(m.hashCode());
- }
}
diff --git a/apps/yang/src/main/java/org/onosproject/yang/YangRuntimeManager.java b/apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java
similarity index 98%
rename from apps/yang/src/main/java/org/onosproject/yang/YangRuntimeManager.java
rename to apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java
index b9ac2e1..5d9a076 100644
--- a/apps/yang/src/main/java/org/onosproject/yang/YangRuntimeManager.java
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/YangRuntimeManager.java
@@ -14,9 +14,8 @@
* limitations under the License.
*/
-package org.onosproject.yang;
+package org.onosproject.yang.impl;
-import com.google.common.annotations.Beta;
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.Deactivate;
@@ -24,6 +23,7 @@
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.apache.felix.scr.annotations.Service;
import org.onosproject.core.CoreService;
+import org.onosproject.yang.YangClassLoaderRegistry;
import org.onosproject.yang.model.ModelConverter;
import org.onosproject.yang.model.ModelObjectData;
import org.onosproject.yang.model.NodeKey;
@@ -62,7 +62,6 @@
/**
* Represents implementation of YANG runtime manager.
*/
-@Beta
@Service
@Component(immediate = true)
public class YangRuntimeManager implements YangModelRegistry,
diff --git a/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java b/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java
new file mode 100755
index 0000000..5140248
--- /dev/null
+++ b/apps/yang/src/main/java/org/onosproject/yang/impl/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementations of various YANG runtime services.
+ */
+package org.onosproject.yang.impl;
\ No newline at end of file