blob: 293f5e9ed501a2e2d81aec95ccc606abdc09f18e [file] [log] [blame]
#!/usr/bin/env python
"""
This script prepares this ONOS directory so that the Sonar Scanner can be run.
- Build ONOS
- Run coverage tests on a per module basis, stage surefire-reports and jacoco.exec
- Generate sonar-project.properties file
"""
import os
import sys
import fnmatch
from shutil import copy, rmtree
from subprocess import check_output, STDOUT, CalledProcessError
ONOS_VERSION = ''
GENFILES = 'bazel-genfiles'
BIN = 'bazel-bin'
SONAR_PROJECT = GENFILES + '/sonar-project'
SUREFIRE_REPORTS = 'surefire-reports'
SONAR_PROPERTIES_FILE_NAME = SONAR_PROJECT + '/sonar-project.properties'
# Template for the sonar properties file
ROOT_TEMPLATE = '''# Auto-generated properties file
sonar.projectKey=%(key)s
sonar.projectName=%(name)s
sonar.projectVersion=%(version)s
sonar.sourceEncoding=UTF-8
sonar.java.target = 1.8
sonar.java.source = 1.8
sonar.language=java
sonar.junit.reportsPath = surefire-reports
sonar.jacoco.reportPath = jacoco.exec
sonar.modules=%(modules)s
'''
def split_target(target):
path, module = target.split(':', 2)
path = path.replace('//', '', 1)
return path, module
def run_command(cmd):
output = check_output(cmd).rstrip()
return output.split('\n') if output else []
def run_command_with_stderr(cmd):
output = check_output(cmd, stderr=STDOUT).rstrip()
return output.split('\n') if output else []
def make_dirs(path):
try:
os.makedirs(path)
except OSError:
pass
def find_bazel_project():
return os.path.basename(os.getcwd())
def find_bazel_classes_directory(module_name, path):
bazel_bin_marker = "bazel-bin:"
bazel_bin_dir = ""
info = run_command(["bazel", "info"])
for bin_dir in info:
if bin_dir.startswith(bazel_bin_marker):
bazel_bin_dir = bin_dir.replace(bazel_bin_marker, "")
break
return bazel_bin_dir
def capture_surefire_reports(module_path):
matches = []
for root, dirnames, filenames in os.walk('bazel-testlogs/' + module_path + '/src/test/java'):
for filename in fnmatch.filter(filenames, '*.xml'):
source_path = os.path.join(root, filename)
matches.append(source_path)
destination_path = \
SONAR_PROJECT + "/" + module_path + "/" + SUREFIRE_REPORTS + "/TEST-" + \
source_path.replace("bazel-testlogs/", "")\
.replace("/test.xml", "test.xml")\
.replace(module_path, "")\
.replace("/src/test/java/", "")\
.replace("/", ".")
make_dirs(path=os.path.dirname(destination_path))
copy(source_path, destination_path)
def capture_jacoco(module_path):
source_path = '/tmp/jacoco.exec'
destination_path = \
SONAR_PROJECT + "/" + module_path
make_dirs(path=os.path.dirname(destination_path))
copy(source_path, destination_path)
def capture_sources(module_path):
source_path = module_path + '/src'
destination_path = SONAR_PROJECT + "/" + module_path + '/src'
os.symlink(os.getcwd() + '/' + source_path, destination_path)
def capture_classes(module_name, path):
# module name: onos-core-net
# path: core/net
base_project_dir = os.getcwd() + "/" + SONAR_PROJECT + "/" + path + "/"
base_bin_dir = os.getcwd() + "/" + BIN + "/" + path + "/lib" + module_name
jar_file_path = base_bin_dir + ".jar"
test_jar_file_path = base_bin_dir + "-tests.jar"
classes_directory = base_project_dir + "classes"
test_classes_directory = base_project_dir + "test-classes"
make_dirs(classes_directory)
current_directory = os.getcwd()
os.chdir(classes_directory)
run_command(["jar", "xvf", jar_file_path])
make_dirs(test_classes_directory)
os.chdir(test_classes_directory)
run_command(["jar", "xvf", test_jar_file_path])
os.chdir(current_directory)
"""
Writes out the properties for a given module and stages the files needed by the scanner.
"""
def write_module(target, out):
path, module_name = split_target(target)
query = 'labels(srcs, "%s-native")' % target
tests_query = 'labels(srcs, "%s-tests")' % target
# get rid of previous data
try:
os.remove('/tmp/jacoco.exec')
except OSError:
pass
try:
# Find all the test targets in this package
coverage_target_query = "attr(name, .*-coverage, tests(//" + path + ":*))"
coverage_targets_result = run_command(["bazel", "query", coverage_target_query])
# Find the test targets that are coverage targets
run_coverage_command = ['bazel', 'test']
for coverage_target in coverage_targets_result:
run_coverage_command.append(str(coverage_target))
except CalledProcessError:
print "Error querying test files for target " + target
return
try:
# Use bazel to run all the coverage targets
run_coverage_command.append('--cache_test_results=no')
run_command(run_coverage_command)
# Find the source files used by the base library
sources = run_command(['bazel', 'query', query])
# Find the source files used by the tests
test_sources = run_command(['bazel', 'query', tests_query])
except CalledProcessError as exc:
print "Error running test files for target " + target
raise exc
if not os.path.exists('/tmp/jacoco.exec'):
# No coverage data was produced, make an empty one
open('/tmp/jacoco.exec', 'a').close()
# Filter out non-Java files
sources = \
[source_file for source_file in sources if "package-info" not in source_file and ".java" in source_file]
test_sources = \
[test_source_file for test_source_file in test_sources
if "package-info" not in test_source_file and ".java" in test_source_file]
# Adjust source file paths to be relative to the root
sources_filtered = []
for source in sources:
sources_filtered.append(source.replace('//' + path + ':', "", 1))
test_sources_filtered = []
for test_source in test_sources:
test_sources_filtered.append(test_source.replace('//' + path + ':', "", 1))
# create a CSL of all the source files for use in the properties file
sources_csl = ",".join(sources_filtered).replace("//", "").replace(":", "/")
# create a CSL of all the test source files for use in the properties file
test_sources_csl = ",".join(test_sources_filtered).replace("//", "").replace(":", "/")
# Write out the properties for this package
out.write('%s.sonar.projectBaseDir=%s\n' % (module_name, path))
out.write('%(name)s.sonar.projectName=%(name)s\n' % {'name': module_name})
out.write('%s.sonar.sources=%s\n' % (module_name, sources_csl))
out.write('%s.sonar.tests=%s\n' % (module_name, test_sources_csl))
out.write('%s.sonar.java.binaries = classes\n' % module_name)
out.write('%s.sonar.java.test.binaries = test-classes\n' % module_name)
# Get the dependencies for this package using bazel
deps_files = run_command_with_stderr(['bazel', 'build', '%s-tests-gen-deps' % target])
dep_file = ""
for source_file in deps_files:
if source_file.endswith(".txt"):
dep_file = source_file.strip()
libraries = []
with open(dep_file, 'r') as read_files:
lines = read_files.readline().split(',')
external_base_path = os.path.realpath(
os.path.realpath("bazel-" + find_bazel_project() + "/external") + "/../../..")
for line in lines:
library_file_name = line
if line.startswith("external"):
library_file_name = external_base_path + "/" + library_file_name
else:
library_file_name = os.getcwd() + "/" + library_file_name
libraries.append(library_file_name)
out.write('%s.sonar.java.libraries = %s\n' % (module_name, ",".join(libraries)))
# Capture files needed by the scanner into the staging area
capture_surefire_reports(path)
capture_jacoco(path)
capture_sources(path)
capture_classes(module_name, path)
'''
Extracts the ONOS version string from the bazel variables.
'''
def _extract_onos_version():
bazel_vars_path = os.getcwd() + "/tools/build/bazel/variables.bzl"
with open(bazel_vars_path, 'r') as bazel_vars_file:
for bazel_var in bazel_vars_file.readlines():
if "ONOS_VERSION" in bazel_var:
return bazel_var.replace(" ", "").replace('ONOS_VERSION='2.1.0-SNAPSHOT'
def _main():
global ONOS_ROOT, ONOS_VERSION, targets
debug = False
if len(sys.argv) > 1:
debug = True
# Change to $ONOS_ROOT
ONOS_ROOT = os.environ['ONOS_ROOT']
if ONOS_ROOT:
os.chdir(ONOS_ROOT)
# extract the ONOS version string
ONOS_VERSION = _extract_onos_version()
# build ONOS
run_command(["bazel", "build", "onos"])
# find the test targets to get coverage for
if debug:
# Use a predefined list of targets for debugging
targets = ['//core/net:onos-core-net', '//utils/misc:onlab-misc']
else:
# Query all onos OSGi jar file rules with tests from bazel
targets = run_command(["bazel", "query", "attr('name', '.*-tests-gen', '//...')"])
targets = [target.replace("-tests-gen-deps", "") for target in targets]
# Filter out targets without any tests in them
targets_with_tests = []
for target in targets:
colon = target.find(':')
base_target = target[0:colon]
target_query_result = run_command(['bazel', 'query', 'tests(' + base_target + ':*)'])
for result_line in target_query_result:
if "src/test" in result_line:
targets_with_tests.append(target)
break
targets = targets_with_tests
# Clear out any old results
rmtree(SONAR_PROJECT, True)
# make a directory for the new results
make_dirs(SONAR_PROJECT)
# fill in the template for the sonar properties
sonar_parameters = {
'name': 'onos',
'key': 'org.onosproject:onos',
'version': ONOS_VERSION,
'jacoco': '/tmp/jacoco.exec',
'reports': 'surefire-reports',
'modules': ','.join([split_target(t)[1] for t in targets])
}
if debug:
sonar_parameters["key"] = 'org.onosproject:onos-test-sonar'
sonar_parameters["name"] = 'onos-test-sonar'
# Write the sonar properties file
with open(SONAR_PROPERTIES_FILE_NAME, 'w') as out:
out.write(ROOT_TEMPLATE % sonar_parameters)
for target in targets:
print "Processing coverage for target " + target
write_module(target, out)
if __name__ == "__main__":
_main()