blob: 293f5e9ed501a2e2d81aec95ccc606abdc09f18e [file] [log] [blame]
Brian O'Connor9c2c8232016-11-08 17:13:14 -08001#!/usr/bin/env python
2
Ray Milkey8705cce2019-01-14 14:05:48 -08003"""
4This script prepares this ONOS directory so that the Sonar Scanner can be run.
5 - Build ONOS
6 - Run coverage tests on a per module basis, stage surefire-reports and jacoco.exec
7 - Generate sonar-project.properties file
8"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -08009
Brian O'Connor9c2c8232016-11-08 17:13:14 -080010import os
Ray Milkey8705cce2019-01-14 14:05:48 -080011import sys
12import fnmatch
Brian O'Connor9c2c8232016-11-08 17:13:14 -080013
Ray Milkey8705cce2019-01-14 14:05:48 -080014from shutil import copy, rmtree
15from subprocess import check_output, STDOUT, CalledProcessError
Brian O'Connor9c2c8232016-11-08 17:13:14 -080016
Ray Milkeycdc94162019-02-07 15:54:08 -080017ONOS_VERSION = ''
Ray Milkey924c0e32016-11-18 13:47:14 -080018
Ray Milkey8705cce2019-01-14 14:05:48 -080019GENFILES = 'bazel-genfiles'
Ray Milkeyfd515152019-01-24 22:02:39 -080020BIN = 'bazel-bin'
21
Ray Milkey8705cce2019-01-14 14:05:48 -080022SONAR_PROJECT = GENFILES + '/sonar-project'
23SUREFIRE_REPORTS = 'surefire-reports'
24SONAR_PROPERTIES_FILE_NAME = SONAR_PROJECT + '/sonar-project.properties'
25
26# Template for the sonar properties file
Brian O'Connor9c2c8232016-11-08 17:13:14 -080027ROOT_TEMPLATE = '''# Auto-generated properties file
28sonar.projectKey=%(key)s
29sonar.projectName=%(name)s
30sonar.projectVersion=%(version)s
Ray Milkey924c0e32016-11-18 13:47:14 -080031
Brian O'Connor9c2c8232016-11-08 17:13:14 -080032sonar.sourceEncoding=UTF-8
33sonar.java.target = 1.8
34sonar.java.source = 1.8
35sonar.language=java
36
Ray Milkey8705cce2019-01-14 14:05:48 -080037sonar.junit.reportsPath = surefire-reports
38sonar.jacoco.reportPath = jacoco.exec
Brian O'Connor9c2c8232016-11-08 17:13:14 -080039
40sonar.modules=%(modules)s
41
42'''
43
Brian O'Connor9c2c8232016-11-08 17:13:14 -080044
Ray Milkey8705cce2019-01-14 14:05:48 -080045def split_target(target):
Ray Milkey32963cf2018-11-21 09:31:51 -080046 path, module = target.split(':', 2)
47 path = path.replace('//', '', 1)
48 return path, module
49
Brian O'Connor9c2c8232016-11-08 17:13:14 -080050
Ray Milkey8705cce2019-01-14 14:05:48 -080051def run_command(cmd):
Ray Milkey32963cf2018-11-21 09:31:51 -080052 output = check_output(cmd).rstrip()
53 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080054
Brian O'Connor9c2c8232016-11-08 17:13:14 -080055
Ray Milkey8705cce2019-01-14 14:05:48 -080056def run_command_with_stderr(cmd):
57 output = check_output(cmd, stderr=STDOUT).rstrip()
58 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080059
Brian O'Connor9c2c8232016-11-08 17:13:14 -080060
Ray Milkey8705cce2019-01-14 14:05:48 -080061def make_dirs(path):
62 try:
63 os.makedirs(path)
64 except OSError:
65 pass
Brian O'Connor9c2c8232016-11-08 17:13:14 -080066
Brian O'Connor9c2c8232016-11-08 17:13:14 -080067
Ray Milkey8705cce2019-01-14 14:05:48 -080068def find_bazel_project():
69 return os.path.basename(os.getcwd())
Brian O'Connor9c2c8232016-11-08 17:13:14 -080070
Ray Milkey8705cce2019-01-14 14:05:48 -080071
72def find_bazel_classes_directory(module_name, path):
Ray Milkeye6b5d9e2019-01-23 13:18:23 -080073 bazel_bin_marker = "bazel-bin:"
74 bazel_bin_dir = ""
75 info = run_command(["bazel", "info"])
76 for bin_dir in info:
77 if bin_dir.startswith(bazel_bin_marker):
78 bazel_bin_dir = bin_dir.replace(bazel_bin_marker, "")
79 break
80 return bazel_bin_dir
Ray Milkey8705cce2019-01-14 14:05:48 -080081
82
83def capture_surefire_reports(module_path):
84 matches = []
85 for root, dirnames, filenames in os.walk('bazel-testlogs/' + module_path + '/src/test/java'):
86 for filename in fnmatch.filter(filenames, '*.xml'):
87 source_path = os.path.join(root, filename)
88 matches.append(source_path)
89
90 destination_path = \
91 SONAR_PROJECT + "/" + module_path + "/" + SUREFIRE_REPORTS + "/TEST-" + \
92 source_path.replace("bazel-testlogs/", "")\
93 .replace("/test.xml", "test.xml")\
94 .replace(module_path, "")\
95 .replace("/src/test/java/", "")\
96 .replace("/", ".")
97 make_dirs(path=os.path.dirname(destination_path))
98 copy(source_path, destination_path)
99
100
101def capture_jacoco(module_path):
102 source_path = '/tmp/jacoco.exec'
103 destination_path = \
104 SONAR_PROJECT + "/" + module_path
105 make_dirs(path=os.path.dirname(destination_path))
106 copy(source_path, destination_path)
107
108
109def capture_sources(module_path):
110 source_path = module_path + '/src'
111 destination_path = SONAR_PROJECT + "/" + module_path + '/src'
112 os.symlink(os.getcwd() + '/' + source_path, destination_path)
113
114
Ray Milkeyfd515152019-01-24 22:02:39 -0800115def capture_classes(module_name, path):
116 # module name: onos-core-net
117 # path: core/net
118
119 base_project_dir = os.getcwd() + "/" + SONAR_PROJECT + "/" + path + "/"
120 base_bin_dir = os.getcwd() + "/" + BIN + "/" + path + "/lib" + module_name
121 jar_file_path = base_bin_dir + ".jar"
122 test_jar_file_path = base_bin_dir + "-tests.jar"
123 classes_directory = base_project_dir + "classes"
124 test_classes_directory = base_project_dir + "test-classes"
125
126 make_dirs(classes_directory)
127 current_directory = os.getcwd()
128 os.chdir(classes_directory)
129 run_command(["jar", "xvf", jar_file_path])
130
131 make_dirs(test_classes_directory)
132 os.chdir(test_classes_directory)
133 run_command(["jar", "xvf", test_jar_file_path])
134
135 os.chdir(current_directory)
136
137
Ray Milkey8705cce2019-01-14 14:05:48 -0800138"""
139 Writes out the properties for a given module and stages the files needed by the scanner.
140"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800141
142
143def write_module(target, out):
Ray Milkey8705cce2019-01-14 14:05:48 -0800144 path, module_name = split_target(target)
145 query = 'labels(srcs, "%s-native")' % target
Ray Milkeyfd515152019-01-24 22:02:39 -0800146 tests_query = 'labels(srcs, "%s-tests")' % target
Ray Milkey8705cce2019-01-14 14:05:48 -0800147
148 # get rid of previous data
149 try:
150 os.remove('/tmp/jacoco.exec')
151 except OSError:
152 pass
153
154 try:
155 # Find all the test targets in this package
156 coverage_target_query = "attr(name, .*-coverage, tests(//" + path + ":*))"
157 coverage_targets_result = run_command(["bazel", "query", coverage_target_query])
158
159 # Find the test targets that are coverage targets
160 run_coverage_command = ['bazel', 'test']
161 for coverage_target in coverage_targets_result:
162 run_coverage_command.append(str(coverage_target))
163 except CalledProcessError:
164 print "Error querying test files for target " + target
165 return
166
167 try:
168 # Use bazel to run all the coverage targets
169 run_coverage_command.append('--cache_test_results=no')
170 run_command(run_coverage_command)
171
Ray Milkeyfd515152019-01-24 22:02:39 -0800172 # Find the source files used by the base library
Ray Milkey8705cce2019-01-14 14:05:48 -0800173 sources = run_command(['bazel', 'query', query])
Ray Milkeyfd515152019-01-24 22:02:39 -0800174
175 # Find the source files used by the tests
176 test_sources = run_command(['bazel', 'query', tests_query])
Ray Milkey8705cce2019-01-14 14:05:48 -0800177 except CalledProcessError as exc:
178 print "Error running test files for target " + target
179 raise exc
180
181 if not os.path.exists('/tmp/jacoco.exec'):
Ray Milkeycdc94162019-02-07 15:54:08 -0800182 # No coverage data was produced, make an empty one
183 open('/tmp/jacoco.exec', 'a').close()
Ray Milkey8705cce2019-01-14 14:05:48 -0800184
185 # Filter out non-Java files
186 sources = \
187 [source_file for source_file in sources if "package-info" not in source_file and ".java" in source_file]
Ray Milkeyfd515152019-01-24 22:02:39 -0800188 test_sources = \
189 [test_source_file for test_source_file in test_sources
190 if "package-info" not in test_source_file and ".java" in test_source_file]
Ray Milkey8705cce2019-01-14 14:05:48 -0800191
192 # Adjust source file paths to be relative to the root
193 sources_filtered = []
194 for source in sources:
195 sources_filtered.append(source.replace('//' + path + ':', "", 1))
196
Ray Milkeyfd515152019-01-24 22:02:39 -0800197 test_sources_filtered = []
198 for test_source in test_sources:
199 test_sources_filtered.append(test_source.replace('//' + path + ':', "", 1))
200
Ray Milkey8705cce2019-01-14 14:05:48 -0800201 # create a CSL of all the source files for use in the properties file
202 sources_csl = ",".join(sources_filtered).replace("//", "").replace(":", "/")
203
Ray Milkeyfd515152019-01-24 22:02:39 -0800204 # create a CSL of all the test source files for use in the properties file
205 test_sources_csl = ",".join(test_sources_filtered).replace("//", "").replace(":", "/")
206
Ray Milkey8705cce2019-01-14 14:05:48 -0800207 # Write out the properties for this package
Ray Milkey32963cf2018-11-21 09:31:51 -0800208 out.write('%s.sonar.projectBaseDir=%s\n' % (module_name, path))
209 out.write('%(name)s.sonar.projectName=%(name)s\n' % {'name': module_name})
Ray Milkey32963cf2018-11-21 09:31:51 -0800210 out.write('%s.sonar.sources=%s\n' % (module_name, sources_csl))
Ray Milkeyfd515152019-01-24 22:02:39 -0800211 out.write('%s.sonar.tests=%s\n' % (module_name, test_sources_csl))
212 out.write('%s.sonar.java.binaries = classes\n' % module_name)
213 out.write('%s.sonar.java.test.binaries = test-classes\n' % module_name)
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800214
Ray Milkey8705cce2019-01-14 14:05:48 -0800215 # Get the dependencies for this package using bazel
216 deps_files = run_command_with_stderr(['bazel', 'build', '%s-tests-gen-deps' % target])
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800217
Ray Milkey8705cce2019-01-14 14:05:48 -0800218 dep_file = ""
219 for source_file in deps_files:
220 if source_file.endswith(".txt"):
221 dep_file = source_file.strip()
222
223 libraries = []
224 with open(dep_file, 'r') as read_files:
225 lines = read_files.readline().split(',')
226
227 external_base_path = os.path.realpath(
228 os.path.realpath("bazel-" + find_bazel_project() + "/external") + "/../../..")
229 for line in lines:
230 library_file_name = line
231 if line.startswith("external"):
232 library_file_name = external_base_path + "/" + library_file_name
233 else:
234 library_file_name = os.getcwd() + "/" + library_file_name
235 libraries.append(library_file_name)
236
237 out.write('%s.sonar.java.libraries = %s\n' % (module_name, ",".join(libraries)))
238
239 # Capture files needed by the scanner into the staging area
240 capture_surefire_reports(path)
241 capture_jacoco(path)
242 capture_sources(path)
Ray Milkeyfd515152019-01-24 22:02:39 -0800243 capture_classes(module_name, path)
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800244
Ray Milkey32963cf2018-11-21 09:31:51 -0800245
Ray Milkeycdc94162019-02-07 15:54:08 -0800246'''
247 Extracts the ONOS version string from the bazel variables.
248'''
249
250
251def _extract_onos_version():
252 bazel_vars_path = os.getcwd() + "/tools/build/bazel/variables.bzl"
253 with open(bazel_vars_path, 'r') as bazel_vars_file:
254 for bazel_var in bazel_vars_file.readlines():
255 if "ONOS_VERSION" in bazel_var:
ONOS Jenkins User961dbde2019-04-12 21:52:16 +0000256 return bazel_var.replace(" ", "").replace('ONOS_VERSION='2.1.0-SNAPSHOT'
Ray Milkeycdc94162019-02-07 15:54:08 -0800257
258
Ray Milkey8705cce2019-01-14 14:05:48 -0800259def _main():
Ray Milkeycdc94162019-02-07 15:54:08 -0800260 global ONOS_ROOT, ONOS_VERSION, targets
Ray Milkey8705cce2019-01-14 14:05:48 -0800261 debug = False
262 if len(sys.argv) > 1:
263 debug = True
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800264
Ray Milkey8705cce2019-01-14 14:05:48 -0800265 # Change to $ONOS_ROOT
266 ONOS_ROOT = os.environ['ONOS_ROOT']
267 if ONOS_ROOT:
268 os.chdir(ONOS_ROOT)
269
Ray Milkeycdc94162019-02-07 15:54:08 -0800270 # extract the ONOS version string
271 ONOS_VERSION = _extract_onos_version()
272
Ray Milkey8705cce2019-01-14 14:05:48 -0800273 # build ONOS
274 run_command(["bazel", "build", "onos"])
275
276 # find the test targets to get coverage for
277 if debug:
278 # Use a predefined list of targets for debugging
279 targets = ['//core/net:onos-core-net', '//utils/misc:onlab-misc']
280 else:
281 # Query all onos OSGi jar file rules with tests from bazel
282 targets = run_command(["bazel", "query", "attr('name', '.*-tests-gen', '//...')"])
283 targets = [target.replace("-tests-gen-deps", "") for target in targets]
284
285 # Filter out targets without any tests in them
286 targets_with_tests = []
287 for target in targets:
288 colon = target.find(':')
289 base_target = target[0:colon]
290 target_query_result = run_command(['bazel', 'query', 'tests(' + base_target + ':*)'])
291 for result_line in target_query_result:
292 if "src/test" in result_line:
293 targets_with_tests.append(target)
294 break
295 targets = targets_with_tests
296
297 # Clear out any old results
298 rmtree(SONAR_PROJECT, True)
299
300 # make a directory for the new results
301 make_dirs(SONAR_PROJECT)
302
303 # fill in the template for the sonar properties
304 sonar_parameters = {
Ray Milkey32963cf2018-11-21 09:31:51 -0800305 'name': 'onos',
306 'key': 'org.onosproject:onos',
307 'version': ONOS_VERSION,
Ray Milkey8705cce2019-01-14 14:05:48 -0800308 'jacoco': '/tmp/jacoco.exec',
309 'reports': 'surefire-reports',
310 'modules': ','.join([split_target(t)[1] for t in targets])
311 }
312 if debug:
313 sonar_parameters["key"] = 'org.onosproject:onos-test-sonar'
314 sonar_parameters["name"] = 'onos-test-sonar'
315
316 # Write the sonar properties file
317 with open(SONAR_PROPERTIES_FILE_NAME, 'w') as out:
318 out.write(ROOT_TEMPLATE % sonar_parameters)
319 for target in targets:
320 print "Processing coverage for target " + target
321 write_module(target, out)
322
323
324if __name__ == "__main__":
325 _main()