blob: bda884b01117bfbbcf313c46c885b2c90ee924ac [file] [log] [blame]
Brian O'Connor9c2c8232016-11-08 17:13:14 -08001#!/usr/bin/env python
2
Ray Milkey8705cce2019-01-14 14:05:48 -08003"""
4This script prepares this ONOS directory so that the Sonar Scanner can be run.
5 - Build ONOS
6 - Run coverage tests on a per module basis, stage surefire-reports and jacoco.exec
7 - Generate sonar-project.properties file
8"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -08009
Brian O'Connor9c2c8232016-11-08 17:13:14 -080010import os
Ray Milkey8705cce2019-01-14 14:05:48 -080011import sys
12import fnmatch
Brian O'Connor9c2c8232016-11-08 17:13:14 -080013
Ray Milkey8705cce2019-01-14 14:05:48 -080014from shutil import copy, rmtree
15from subprocess import check_output, STDOUT, CalledProcessError
Brian O'Connor9c2c8232016-11-08 17:13:14 -080016
Ray Milkey32963cf2018-11-21 09:31:51 -080017ONOS_VERSION = '2.0.0-SNAPSHOT'
Ray Milkey924c0e32016-11-18 13:47:14 -080018
Ray Milkey8705cce2019-01-14 14:05:48 -080019GENFILES = 'bazel-genfiles'
20SONAR_PROJECT = GENFILES + '/sonar-project'
21SUREFIRE_REPORTS = 'surefire-reports'
22SONAR_PROPERTIES_FILE_NAME = SONAR_PROJECT + '/sonar-project.properties'
23
24# Template for the sonar properties file
Brian O'Connor9c2c8232016-11-08 17:13:14 -080025ROOT_TEMPLATE = '''# Auto-generated properties file
26sonar.projectKey=%(key)s
27sonar.projectName=%(name)s
28sonar.projectVersion=%(version)s
Ray Milkey924c0e32016-11-18 13:47:14 -080029
Brian O'Connor9c2c8232016-11-08 17:13:14 -080030sonar.sourceEncoding=UTF-8
31sonar.java.target = 1.8
32sonar.java.source = 1.8
33sonar.language=java
34
Ray Milkey8705cce2019-01-14 14:05:48 -080035sonar.junit.reportsPath = surefire-reports
36sonar.jacoco.reportPath = jacoco.exec
Brian O'Connor9c2c8232016-11-08 17:13:14 -080037
38sonar.modules=%(modules)s
39
40'''
41
Brian O'Connor9c2c8232016-11-08 17:13:14 -080042
Ray Milkey8705cce2019-01-14 14:05:48 -080043def split_target(target):
Ray Milkey32963cf2018-11-21 09:31:51 -080044 path, module = target.split(':', 2)
45 path = path.replace('//', '', 1)
46 return path, module
47
Brian O'Connor9c2c8232016-11-08 17:13:14 -080048
Ray Milkey8705cce2019-01-14 14:05:48 -080049def run_command(cmd):
Ray Milkey32963cf2018-11-21 09:31:51 -080050 output = check_output(cmd).rstrip()
51 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080052
Brian O'Connor9c2c8232016-11-08 17:13:14 -080053
Ray Milkey8705cce2019-01-14 14:05:48 -080054def run_command_with_stderr(cmd):
55 output = check_output(cmd, stderr=STDOUT).rstrip()
56 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080057
Brian O'Connor9c2c8232016-11-08 17:13:14 -080058
Ray Milkey8705cce2019-01-14 14:05:48 -080059def make_dirs(path):
60 try:
61 os.makedirs(path)
62 except OSError:
63 pass
Brian O'Connor9c2c8232016-11-08 17:13:14 -080064
Brian O'Connor9c2c8232016-11-08 17:13:14 -080065
Ray Milkey8705cce2019-01-14 14:05:48 -080066def find_bazel_project():
67 return os.path.basename(os.getcwd())
Brian O'Connor9c2c8232016-11-08 17:13:14 -080068
Ray Milkey8705cce2019-01-14 14:05:48 -080069
70def find_bazel_classes_directory(module_name, path):
Ray Milkeye6b5d9e2019-01-23 13:18:23 -080071 bazel_bin_marker = "bazel-bin:"
72 bazel_bin_dir = ""
73 info = run_command(["bazel", "info"])
74 for bin_dir in info:
75 if bin_dir.startswith(bazel_bin_marker):
76 bazel_bin_dir = bin_dir.replace(bazel_bin_marker, "")
77 break
78 return bazel_bin_dir
Ray Milkey8705cce2019-01-14 14:05:48 -080079
80
81def capture_surefire_reports(module_path):
82 matches = []
83 for root, dirnames, filenames in os.walk('bazel-testlogs/' + module_path + '/src/test/java'):
84 for filename in fnmatch.filter(filenames, '*.xml'):
85 source_path = os.path.join(root, filename)
86 matches.append(source_path)
87
88 destination_path = \
89 SONAR_PROJECT + "/" + module_path + "/" + SUREFIRE_REPORTS + "/TEST-" + \
90 source_path.replace("bazel-testlogs/", "")\
91 .replace("/test.xml", "test.xml")\
92 .replace(module_path, "")\
93 .replace("/src/test/java/", "")\
94 .replace("/", ".")
95 make_dirs(path=os.path.dirname(destination_path))
96 copy(source_path, destination_path)
97
98
99def capture_jacoco(module_path):
100 source_path = '/tmp/jacoco.exec'
101 destination_path = \
102 SONAR_PROJECT + "/" + module_path
103 make_dirs(path=os.path.dirname(destination_path))
104 copy(source_path, destination_path)
105
106
107def capture_sources(module_path):
108 source_path = module_path + '/src'
109 destination_path = SONAR_PROJECT + "/" + module_path + '/src'
110 os.symlink(os.getcwd() + '/' + source_path, destination_path)
111
112
113"""
114 Writes out the properties for a given module and stages the files needed by the scanner.
115"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800116
117
118def write_module(target, out):
Ray Milkey8705cce2019-01-14 14:05:48 -0800119 path, module_name = split_target(target)
120 query = 'labels(srcs, "%s-native")' % target
121
122 # get rid of previous data
123 try:
124 os.remove('/tmp/jacoco.exec')
125 except OSError:
126 pass
127
128 try:
129 # Find all the test targets in this package
130 coverage_target_query = "attr(name, .*-coverage, tests(//" + path + ":*))"
131 coverage_targets_result = run_command(["bazel", "query", coverage_target_query])
132
133 # Find the test targets that are coverage targets
134 run_coverage_command = ['bazel', 'test']
135 for coverage_target in coverage_targets_result:
136 run_coverage_command.append(str(coverage_target))
137 except CalledProcessError:
138 print "Error querying test files for target " + target
139 return
140
141 try:
142 # Use bazel to run all the coverage targets
143 run_coverage_command.append('--cache_test_results=no')
144 run_command(run_coverage_command)
145
146 # Find the source files used by the tests
147 sources = run_command(['bazel', 'query', query])
148 except CalledProcessError as exc:
149 print "Error running test files for target " + target
150 raise exc
151
152 if not os.path.exists('/tmp/jacoco.exec'):
153 # No coverage data was produced, not much to do
154 return
155
156 # Filter out non-Java files
157 sources = \
158 [source_file for source_file in sources if "package-info" not in source_file and ".java" in source_file]
159
160 # Adjust source file paths to be relative to the root
161 sources_filtered = []
162 for source in sources:
163 sources_filtered.append(source.replace('//' + path + ':', "", 1))
164
165 # create a CSL of all the source files for use in the properties file
166 sources_csl = ",".join(sources_filtered).replace("//", "").replace(":", "/")
167
168 # Write out the properties for this package
Ray Milkey32963cf2018-11-21 09:31:51 -0800169 out.write('%s.sonar.projectBaseDir=%s\n' % (module_name, path))
170 out.write('%(name)s.sonar.projectName=%(name)s\n' % {'name': module_name})
Ray Milkey32963cf2018-11-21 09:31:51 -0800171 out.write('%s.sonar.sources=%s\n' % (module_name, sources_csl))
Ray Milkey8705cce2019-01-14 14:05:48 -0800172 binaries = find_bazel_classes_directory(module_name, path)
173 out.write('%s.sonar.java.binaries = %s\n' % (module_name, binaries))
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800174
Ray Milkey8705cce2019-01-14 14:05:48 -0800175 # Get the dependencies for this package using bazel
176 deps_files = run_command_with_stderr(['bazel', 'build', '%s-tests-gen-deps' % target])
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800177
Ray Milkey8705cce2019-01-14 14:05:48 -0800178 dep_file = ""
179 for source_file in deps_files:
180 if source_file.endswith(".txt"):
181 dep_file = source_file.strip()
182
183 libraries = []
184 with open(dep_file, 'r') as read_files:
185 lines = read_files.readline().split(',')
186
187 external_base_path = os.path.realpath(
188 os.path.realpath("bazel-" + find_bazel_project() + "/external") + "/../../..")
189 for line in lines:
190 library_file_name = line
191 if line.startswith("external"):
192 library_file_name = external_base_path + "/" + library_file_name
193 else:
194 library_file_name = os.getcwd() + "/" + library_file_name
195 libraries.append(library_file_name)
196
197 out.write('%s.sonar.java.libraries = %s\n' % (module_name, ",".join(libraries)))
198
199 # Capture files needed by the scanner into the staging area
200 capture_surefire_reports(path)
201 capture_jacoco(path)
202 capture_sources(path)
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800203
Ray Milkey32963cf2018-11-21 09:31:51 -0800204
Ray Milkey8705cce2019-01-14 14:05:48 -0800205def _main():
206 global ONOS_ROOT, targets
207 debug = False
208 if len(sys.argv) > 1:
209 debug = True
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800210
Ray Milkey8705cce2019-01-14 14:05:48 -0800211 # Change to $ONOS_ROOT
212 ONOS_ROOT = os.environ['ONOS_ROOT']
213 if ONOS_ROOT:
214 os.chdir(ONOS_ROOT)
215
216 # build ONOS
217 run_command(["bazel", "build", "onos"])
218
219 # find the test targets to get coverage for
220 if debug:
221 # Use a predefined list of targets for debugging
222 targets = ['//core/net:onos-core-net', '//utils/misc:onlab-misc']
223 else:
224 # Query all onos OSGi jar file rules with tests from bazel
225 targets = run_command(["bazel", "query", "attr('name', '.*-tests-gen', '//...')"])
226 targets = [target.replace("-tests-gen-deps", "") for target in targets]
227
228 # Filter out targets without any tests in them
229 targets_with_tests = []
230 for target in targets:
231 colon = target.find(':')
232 base_target = target[0:colon]
233 target_query_result = run_command(['bazel', 'query', 'tests(' + base_target + ':*)'])
234 for result_line in target_query_result:
235 if "src/test" in result_line:
236 targets_with_tests.append(target)
237 break
238 targets = targets_with_tests
239
240 # Clear out any old results
241 rmtree(SONAR_PROJECT, True)
242
243 # make a directory for the new results
244 make_dirs(SONAR_PROJECT)
245
246 # fill in the template for the sonar properties
247 sonar_parameters = {
Ray Milkey32963cf2018-11-21 09:31:51 -0800248 'name': 'onos',
249 'key': 'org.onosproject:onos',
250 'version': ONOS_VERSION,
Ray Milkey8705cce2019-01-14 14:05:48 -0800251 'jacoco': '/tmp/jacoco.exec',
252 'reports': 'surefire-reports',
253 'modules': ','.join([split_target(t)[1] for t in targets])
254 }
255 if debug:
256 sonar_parameters["key"] = 'org.onosproject:onos-test-sonar'
257 sonar_parameters["name"] = 'onos-test-sonar'
258
259 # Write the sonar properties file
260 with open(SONAR_PROPERTIES_FILE_NAME, 'w') as out:
261 out.write(ROOT_TEMPLATE % sonar_parameters)
262 for target in targets:
263 print "Processing coverage for target " + target
264 write_module(target, out)
265
266
267if __name__ == "__main__":
268 _main()