Add support to propagate ONOS log to Kafka with KafkaAppender in log4j2

- new OSGi Bundle to define extra imports for log4j2
- update OSGiWrapper to support fragment-host tag + bugfix
- add kafka-clients to onos-dependencies package
- add example of how to propagate log to Kafka via KafkaAppender
- bugfix to org.ops4j.pax.logging.cfg

Change-Id: I89992936101a48c6452082bf23c7133c20aa0b48
diff --git a/BUILD b/BUILD
index a4f5688..626eeee 100644
--- a/BUILD
+++ b/BUILD
@@ -42,12 +42,15 @@
 
 BRANDING = "//tools/package/branding:onos-tools-package-branding"
 
+LOG4J_EXTRA = "//tools/package/log4j2-extra:onos-log4j2-extra"
+
 # Generates auxiliary karaf.zip file; branded and augmented with ONOS runtime tools
 genrule(
     name = "onos-karaf",
     srcs = [
         KARAF,
         BRANDING,
+        LOG4J_EXTRA,
     ] + glob([
         "tools/package/bin/*",
         "tools/package/etc/*",
@@ -55,8 +58,8 @@
         "tools/package/runtime/bin/*",
     ]),
     outs = ["karaf.zip"],
-    cmd = "$(location tools/package/onos-prep-karaf) $(location karaf.zip) $(location %s) %s $(location %s) '' tools/package" %
-          (KARAF, ONOS_VERSION, BRANDING),
+    cmd = "$(location tools/package/onos-prep-karaf) $(location karaf.zip) $(location %s) %s $(location %s) '' $(location %s) tools/package" %
+          (KARAF, ONOS_VERSION, BRANDING, LOG4J_EXTRA),
     tools = ["tools/package/onos-prep-karaf"],
 )
 
diff --git a/tools/build/bazel/osgi_java_library.bzl b/tools/build/bazel/osgi_java_library.bzl
index 0ca2047..491444f 100644
--- a/tools/build/bazel/osgi_java_library.bzl
+++ b/tools/build/bazel/osgi_java_library.bzl
@@ -85,6 +85,7 @@
     web_xml = ctx.attr.web_xml
     dynamicimportPackages = ""
     karaf_commands = ctx.attr.karaf_commands
+    fragment_host = ctx.attr.fragment_host
     cp = ""
 
     inputDependencies = [input_file]
@@ -122,6 +123,7 @@
         "classes",
         bundle_classpath,
         karaf_commands,
+        fragment_host,
     ]
 
     ctx.actions.run(
@@ -155,6 +157,7 @@
         "web_xml": attr.label_list(allow_files = True),
         "include_resources": attr.string(),
         "karaf_commands": attr.string(),
+        "fragment_host": attr.string(),
         "_bnd_exe": attr.label(
             executable = True,
             cfg = "host",
@@ -366,7 +369,8 @@
         group = "org.onosproject",
         import_packages = "*",
         visibility = ["//visibility:private"],
-        generate_pom = False):
+        generate_pom = False,
+        fragment_host = ""):
     _bnd(
         name = name,
         source = jar,
@@ -376,6 +380,7 @@
         visibility = visibility,
         import_packages = import_packages,
         web_xml = None,
+        fragment_host = fragment_host,
     )
 
     if generate_pom:
diff --git a/tools/package/dependencies/BUILD b/tools/package/dependencies/BUILD
index 63adf9c..bb50031 100644
--- a/tools/package/dependencies/BUILD
+++ b/tools/package/dependencies/BUILD
@@ -28,6 +28,7 @@
     "@org_apache_karaf_shell_console//jar",
     "@org_osgi_cmpn//jar",
     "@jersey_container_servlet//jar",
+    "@kafka_clients//jar",
 ]
 
 # Listed in the <dependencies> section with scope 'provided'
diff --git a/tools/package/etc/org.ops4j.pax.logging.cfg b/tools/package/etc/org.ops4j.pax.logging.cfg
index 036447d..c771aa9 100644
--- a/tools/package/etc/org.ops4j.pax.logging.cfg
+++ b/tools/package/etc/org.ops4j.pax.logging.cfg
@@ -36,13 +36,12 @@
 #log4j2.rootLogger.type = asyncRoot
 #log4j2.rootLogger.includeLocation = false
 log4j2.rootLogger.appenderRef.RollingFile.ref = RollingFile
-log4j2.rootLogger.appenderRef.AuditFile.ref = AuditFile
 log4j2.rootLogger.appenderRef.PaxOsgi.ref = PaxOsgi
 log4j2.rootLogger.appenderRef.Console.ref = Console
 log4j2.rootLogger.appenderRef.Console.filter.regex.type = RegexFilter
 log4j2.rootLogger.appenderRef.Console.filter.regex.regex = .*Audit.*
 log4j2.rootLogger.appenderRef.Console.filter.regex.onMatch = DENY
-log4j2.rootLogger.appenderRef.Console.filter.regex.onMisMatch = ACCEPT
+log4j2.rootLogger.appenderRef.Console.filter.regex.onMismatch = ACCEPT
 #log4j2.rootLogger.appenderRef.Console.filter.threshold.type = ThresholdFilter
 #log4j2.rootLogger.appenderRef.Console.filter.threshold.level = ${karaf.log.console:-OFF}
 
@@ -86,7 +85,7 @@
 log4j2.appender.rolling.filter.regex.type = RegexFilter
 log4j2.appender.rolling.filter.regex.regex = .*AuditLog.*
 log4j2.appender.rolling.filter.regex.onMatch = DENY
-log4j2.appender.rolling.filter.regex.onMisMatch = ACCEPT
+log4j2.appender.rolling.filter.regex.onMismatch = ACCEPT
 log4j2.appender.rolling.fileName = ${karaf.data}/log/karaf.log
 log4j2.appender.rolling.filePattern = ${karaf.data}/log/karaf.log.%i
 # uncomment to not force a disk flush
@@ -144,3 +143,22 @@
 #log4j2.logger.http-headers.level = DEBUG
 #log4j2.logger.maven.name = org.ops4j.pax.url.mvn
 #log4j2.logger.maven.level = TRACE
+
+# --- Example for a Kafka appender ---
+# Avoid recursive logging for Kafka
+# log4j2.logger.apacheKafka.name = org.apache.kafka
+# log4j2.logger.apacheKafka.level = INFO
+
+# Kafka appender definition
+# log4j2.appender.kafka.type = Kafka
+# log4j2.appender.kafka.name = Kafka
+# log4j2.appender.kafka.property.type = Property
+# log4j2.appender.kafka.property.name = bootstrap.servers
+# log4j2.appender.kafka.property.value = localhost:9092
+# log4j2.appender.kafka.topic = onos.log
+# Async send, no need to wait for Kafka ack for each record
+# log4j2.appender.kafka.syncSend = false
+# log4j2.kafka.pattern = {"@timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}","levelname":"%p","threadName":"%t","category":"%c{1}","bundle.id":"%X{bundle.id}","bundle.name":"%X{bundle.name}","bundle.version":"%X{bundle.version}","message":"%m"}%n
+# log4j2.appender.kafka.layout.type = PatternLayout
+# log4j2.appender.kafka.layout.pattern = ${log4j2.kafka.pattern}
+# log4j2.rootLogger.appenderRef.Kafka.ref = Kafka
\ No newline at end of file
diff --git a/tools/package/etc/startup.properties b/tools/package/etc/startup.properties
new file mode 100644
index 0000000..086936d
--- /dev/null
+++ b/tools/package/etc/startup.properties
@@ -0,0 +1,23 @@
+# Valid for Karaf 4.2.8. If updating Karaf version in ONOS, make sure to
+# copy-paste here the content of apache-karaf-*/etc/startup.properties in, and
+# update the Kafka and log4j2 additions at the end if needed
+
+# Standard startup.properties file
+# Bundles to be started on startup, with startlevel
+mvn\:org.apache.karaf.features/org.apache.karaf.features.extension/4.2.8 = 1
+mvn\:org.apache.karaf.services/org.apache.karaf.services.eventadmin/4.2.8 = 5
+mvn\:org.ops4j.pax.url/pax-url-aether/2.6.2 = 5
+mvn\:org.apache.felix/org.apache.felix.metatype/1.2.2 = 5
+mvn\:org.ops4j.pax.logging/pax-logging-api/1.11.4 = 8
+mvn\:org.fusesource.jansi/jansi/1.18 = 8
+mvn\:org.ops4j.pax.logging/pax-logging-log4j2/1.11.4 = 8
+mvn\:org.apache.felix/org.apache.felix.coordinator/1.0.2 = 9
+mvn\:org.apache.felix/org.apache.felix.configadmin/1.9.16 = 10
+mvn\:org.apache.felix/org.apache.felix.fileinstall/3.6.4 = 11
+mvn\:org.apache.karaf.features/org.apache.karaf.features.core/4.2.8 = 15
+
+# Added part to enable Kafka log4j2 appender
+# N.B.: kafka-clients should have startlevel lower than onos-tools-package-log4j2-extra
+mvn\:org.apache.servicemix.bundles/org.apache.servicemix.bundles.kafka-clients/1.1.1_1 = 6
+# N.B.: onos-tools-package-log4j2-extra should have lower startlevel than pax-logging-*
+mvn\:org.onosproject/onos-log4j2-extra/$ONOS_VERSION = 7
diff --git a/tools/package/log4j2-extra/BUILD b/tools/package/log4j2-extra/BUILD
new file mode 100644
index 0000000..d931f30
--- /dev/null
+++ b/tools/package/log4j2-extra/BUILD
@@ -0,0 +1,28 @@
+load("//tools/build/bazel:osgi_java_library.bzl", "wrapped_osgi_jar")
+
+# This OSGi bundle is needed to import extra packages on log4j2 library
+
+# To include other packages, specify them as comma separated list.
+# As reference for packages that can be imported see:
+# https://github.com/ops4j/org.ops4j.pax.logging/blob/master/pax-logging-log4j2-extra/osgi.bnd
+IMPORT_PACKAGES = "org.apache.kafka.clients.producer,org.apache.kafka.common.serialization"
+
+# An empty OSGi jar bundle including a Manifest that will be merged with
+# the specified fragment_host bundle.
+wrapped_osgi_jar(
+    name = "onos-log4j2-extra",
+    fragment_host = "org.ops4j.pax.logging.pax-logging-log4j2",
+    generate_pom = False,
+    import_packages = IMPORT_PACKAGES,
+    jar = ":empty-log4j2-extra-jar",
+    visibility = ["//visibility:public"],
+    deps = [],
+)
+
+# An empty jar so we can wrap it inside an OSGi bundle.
+# The parameter resource_strip_prefix is needed to not export packages in the OSGi bundle.
+java_library(
+    name = "empty-log4j2-extra-jar",
+    resource_strip_prefix = "tools/package/log4j2-extra",
+    resources = ["empty.txt"],
+)
diff --git a/tools/package/log4j2-extra/empty.txt b/tools/package/log4j2-extra/empty.txt
new file mode 100644
index 0000000..c1d77b7
--- /dev/null
+++ b/tools/package/log4j2-extra/empty.txt
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//This jar is intentionally left empty
\ No newline at end of file
diff --git a/tools/package/onos-prep-karaf b/tools/package/onos-prep-karaf
index 315aa13..5660358 100755
--- a/tools/package/onos-prep-karaf
+++ b/tools/package/onos-prep-karaf
@@ -10,7 +10,8 @@
 ONOS_VERSION=$3
 BRANDING=$4
 KARAF_PATCHES=$5
-SANDBOX=${6:-.}
+LOG4J2_EXTRA=$6
+SANDBOX=${7:-.}
 
 KARAF_VERSION="4.2.8"
 ONOS_SECURITY_MODE="false"
@@ -55,8 +56,16 @@
 cp -r $SANDBOX/init $PREFIX
 cp -r $SANDBOX/etc/* $PREFIX/$KARAF_DIR/etc/
 
+ONOS_VERSION_POINT=$(echo ${ONOS_VERSION} | sed -E 's/-/./')
+
 # Fix the onos version string in the features config
 sed -i".VERBACK" -E "s/.ONOS_VERSION/${ONOS_VERSION}/" $PREFIX/$KARAF_DIR/etc/org.apache.karaf.features.cfg
+# Fix onos version in startup properties
+sed -i".VERBACK" "s/.ONOS_VERSION/${ONOS_VERSION_POINT}/" $PREFIX/$KARAF_DIR/etc/startup.properties
+
+# Add log4j2-extra Bundle
+mkdir -p $PREFIX/$KARAF_DIR/system/org/onosproject/onos-log4j2-extra/$ONOS_VERSION_POINT/
+cp $LOG4J2_EXTRA $PREFIX/$KARAF_DIR/system/org/onosproject/onos-log4j2-extra/$ONOS_VERSION_POINT/onos-log4j2-extra-$ONOS_VERSION_POINT.jar
 
 if [ "$ONOS_SECURITY_MODE" = true ]
 then
diff --git a/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java b/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java
index b57dab5..2bdc2b3 100644
--- a/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java
+++ b/utils/osgiwrap/src/main/java/org/onlab/osgiwrap/OSGiWrapper.java
@@ -86,9 +86,11 @@
     private String bundleClasspath;
     private String karafCommands;
 
+    private String fragmentHost;
+
     // FIXME should consider using Commons CLI, etc.
     public static void main(String[] args) {
-        if (args.length < 14) {
+        if (args.length < 17) {
             System.err.println("Not enough args");
             System.exit(1);
         }
@@ -109,7 +111,8 @@
         String destdir = args[13];
         String bundleClasspath = args[14];
         String karafCommands = args[15];
-        String desc = Joiner.on(' ').join(Arrays.copyOfRange(args, 12, args.length));
+        String fragmentHost = args[16];
+        String desc = Joiner.on(' ').join(Arrays.copyOfRange(args, 17, args.length));
 
         OSGiWrapper wrapper = new OSGiWrapper(jar, output, cp,
                 name, group,
@@ -122,7 +125,8 @@
                 desc,
                 destdir,
                 bundleClasspath,
-                karafCommands);
+                karafCommands,
+                fragmentHost);
         wrapper.log(wrapper + "\n");
         if (!wrapper.execute()) {
             System.err.printf("Error generating %s\n", name);
@@ -147,7 +151,8 @@
                        String bundleDescription,
                        String destdir,
                        String bundleClasspath,
-                       String karafCommands) {
+                       String karafCommands,
+                       String fragmentHost) {
         this.inputJar = inputJar;
         this.classpath = Lists.newArrayList(classpath.split(":"));
         if (!this.classpath.contains(inputJar)) {
@@ -179,6 +184,8 @@
 
         this.bundleClasspath = bundleClasspath;
         this.karafCommands = karafCommands;
+
+        this.fragmentHost = fragmentHost;
     }
 
     private void setProperties(Analyzer analyzer) {
@@ -214,6 +221,8 @@
                     ",org.glassfish.jersey.servlet,org.jvnet.mimepull\n");
         }
         analyzer.setProperty("Karaf-Commands", karafCommands);
+
+        analyzer.setProperty(Analyzer.FRAGMENT_HOST, fragmentHost);
     }
 
     public boolean execute() {