blob: 758693a36e7ff472215a0e5d26fa8d1bd2701060 [file] [log] [blame]
Brian O'Connor9c2c8232016-11-08 17:13:14 -08001#!/usr/bin/env python
2
Ray Milkey8705cce2019-01-14 14:05:48 -08003"""
4This script prepares this ONOS directory so that the Sonar Scanner can be run.
5 - Build ONOS
6 - Run coverage tests on a per module basis, stage surefire-reports and jacoco.exec
7 - Generate sonar-project.properties file
8"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -08009
Brian O'Connor9c2c8232016-11-08 17:13:14 -080010import os
Ray Milkey8705cce2019-01-14 14:05:48 -080011import sys
12import fnmatch
Brian O'Connor9c2c8232016-11-08 17:13:14 -080013
Ray Milkey8705cce2019-01-14 14:05:48 -080014from shutil import copy, rmtree
15from subprocess import check_output, STDOUT, CalledProcessError
Brian O'Connor9c2c8232016-11-08 17:13:14 -080016
Ray Milkey32963cf2018-11-21 09:31:51 -080017ONOS_VERSION = '2.0.0-SNAPSHOT'
Ray Milkey924c0e32016-11-18 13:47:14 -080018
Ray Milkey8705cce2019-01-14 14:05:48 -080019GENFILES = 'bazel-genfiles'
20SONAR_PROJECT = GENFILES + '/sonar-project'
21SUREFIRE_REPORTS = 'surefire-reports'
22SONAR_PROPERTIES_FILE_NAME = SONAR_PROJECT + '/sonar-project.properties'
23
24# Template for the sonar properties file
Brian O'Connor9c2c8232016-11-08 17:13:14 -080025ROOT_TEMPLATE = '''# Auto-generated properties file
26sonar.projectKey=%(key)s
27sonar.projectName=%(name)s
28sonar.projectVersion=%(version)s
Ray Milkey924c0e32016-11-18 13:47:14 -080029
Brian O'Connor9c2c8232016-11-08 17:13:14 -080030sonar.sourceEncoding=UTF-8
31sonar.java.target = 1.8
32sonar.java.source = 1.8
33sonar.language=java
34
Ray Milkey8705cce2019-01-14 14:05:48 -080035sonar.junit.reportsPath = surefire-reports
36sonar.jacoco.reportPath = jacoco.exec
Brian O'Connor9c2c8232016-11-08 17:13:14 -080037
38sonar.modules=%(modules)s
39
40'''
41
Brian O'Connor9c2c8232016-11-08 17:13:14 -080042
Ray Milkey8705cce2019-01-14 14:05:48 -080043def split_target(target):
Ray Milkey32963cf2018-11-21 09:31:51 -080044 path, module = target.split(':', 2)
45 path = path.replace('//', '', 1)
46 return path, module
47
Brian O'Connor9c2c8232016-11-08 17:13:14 -080048
Ray Milkey8705cce2019-01-14 14:05:48 -080049def run_command(cmd):
Ray Milkey32963cf2018-11-21 09:31:51 -080050 output = check_output(cmd).rstrip()
51 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080052
Brian O'Connor9c2c8232016-11-08 17:13:14 -080053
Ray Milkey8705cce2019-01-14 14:05:48 -080054def run_command_with_stderr(cmd):
55 output = check_output(cmd, stderr=STDOUT).rstrip()
56 return output.split('\n') if output else []
Brian O'Connor9c2c8232016-11-08 17:13:14 -080057
Brian O'Connor9c2c8232016-11-08 17:13:14 -080058
Ray Milkey8705cce2019-01-14 14:05:48 -080059def make_dirs(path):
60 try:
61 os.makedirs(path)
62 except OSError:
63 pass
Brian O'Connor9c2c8232016-11-08 17:13:14 -080064
Brian O'Connor9c2c8232016-11-08 17:13:14 -080065
Ray Milkey8705cce2019-01-14 14:05:48 -080066def find_bazel_project():
67 return os.path.basename(os.getcwd())
Brian O'Connor9c2c8232016-11-08 17:13:14 -080068
Ray Milkey8705cce2019-01-14 14:05:48 -080069
70def find_bazel_classes_directory(module_name, path):
71 return os.getcwd() + "/bazel-out/darwin-fastbuild/bin/" + path + \
72 "/_javac/" + module_name + "-native/lib" + module_name + "-native-class_classes"
73
74
75def capture_surefire_reports(module_path):
76 matches = []
77 for root, dirnames, filenames in os.walk('bazel-testlogs/' + module_path + '/src/test/java'):
78 for filename in fnmatch.filter(filenames, '*.xml'):
79 source_path = os.path.join(root, filename)
80 matches.append(source_path)
81
82 destination_path = \
83 SONAR_PROJECT + "/" + module_path + "/" + SUREFIRE_REPORTS + "/TEST-" + \
84 source_path.replace("bazel-testlogs/", "")\
85 .replace("/test.xml", "test.xml")\
86 .replace(module_path, "")\
87 .replace("/src/test/java/", "")\
88 .replace("/", ".")
89 make_dirs(path=os.path.dirname(destination_path))
90 copy(source_path, destination_path)
91
92
93def capture_jacoco(module_path):
94 source_path = '/tmp/jacoco.exec'
95 destination_path = \
96 SONAR_PROJECT + "/" + module_path
97 make_dirs(path=os.path.dirname(destination_path))
98 copy(source_path, destination_path)
99
100
101def capture_sources(module_path):
102 source_path = module_path + '/src'
103 destination_path = SONAR_PROJECT + "/" + module_path + '/src'
104 os.symlink(os.getcwd() + '/' + source_path, destination_path)
105
106
107"""
108 Writes out the properties for a given module and stages the files needed by the scanner.
109"""
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800110
111
112def write_module(target, out):
Ray Milkey8705cce2019-01-14 14:05:48 -0800113 path, module_name = split_target(target)
114 query = 'labels(srcs, "%s-native")' % target
115
116 # get rid of previous data
117 try:
118 os.remove('/tmp/jacoco.exec')
119 except OSError:
120 pass
121
122 try:
123 # Find all the test targets in this package
124 coverage_target_query = "attr(name, .*-coverage, tests(//" + path + ":*))"
125 coverage_targets_result = run_command(["bazel", "query", coverage_target_query])
126
127 # Find the test targets that are coverage targets
128 run_coverage_command = ['bazel', 'test']
129 for coverage_target in coverage_targets_result:
130 run_coverage_command.append(str(coverage_target))
131 except CalledProcessError:
132 print "Error querying test files for target " + target
133 return
134
135 try:
136 # Use bazel to run all the coverage targets
137 run_coverage_command.append('--cache_test_results=no')
138 run_command(run_coverage_command)
139
140 # Find the source files used by the tests
141 sources = run_command(['bazel', 'query', query])
142 except CalledProcessError as exc:
143 print "Error running test files for target " + target
144 raise exc
145
146 if not os.path.exists('/tmp/jacoco.exec'):
147 # No coverage data was produced, not much to do
148 return
149
150 # Filter out non-Java files
151 sources = \
152 [source_file for source_file in sources if "package-info" not in source_file and ".java" in source_file]
153
154 # Adjust source file paths to be relative to the root
155 sources_filtered = []
156 for source in sources:
157 sources_filtered.append(source.replace('//' + path + ':', "", 1))
158
159 # create a CSL of all the source files for use in the properties file
160 sources_csl = ",".join(sources_filtered).replace("//", "").replace(":", "/")
161
162 # Write out the properties for this package
Ray Milkey32963cf2018-11-21 09:31:51 -0800163 out.write('%s.sonar.projectBaseDir=%s\n' % (module_name, path))
164 out.write('%(name)s.sonar.projectName=%(name)s\n' % {'name': module_name})
Ray Milkey32963cf2018-11-21 09:31:51 -0800165 out.write('%s.sonar.sources=%s\n' % (module_name, sources_csl))
Ray Milkey8705cce2019-01-14 14:05:48 -0800166 binaries = find_bazel_classes_directory(module_name, path)
167 out.write('%s.sonar.java.binaries = %s\n' % (module_name, binaries))
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800168
Ray Milkey8705cce2019-01-14 14:05:48 -0800169 # Get the dependencies for this package using bazel
170 deps_files = run_command_with_stderr(['bazel', 'build', '%s-tests-gen-deps' % target])
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800171
Ray Milkey8705cce2019-01-14 14:05:48 -0800172 dep_file = ""
173 for source_file in deps_files:
174 if source_file.endswith(".txt"):
175 dep_file = source_file.strip()
176
177 libraries = []
178 with open(dep_file, 'r') as read_files:
179 lines = read_files.readline().split(',')
180
181 external_base_path = os.path.realpath(
182 os.path.realpath("bazel-" + find_bazel_project() + "/external") + "/../../..")
183 for line in lines:
184 library_file_name = line
185 if line.startswith("external"):
186 library_file_name = external_base_path + "/" + library_file_name
187 else:
188 library_file_name = os.getcwd() + "/" + library_file_name
189 libraries.append(library_file_name)
190
191 out.write('%s.sonar.java.libraries = %s\n' % (module_name, ",".join(libraries)))
192
193 # Capture files needed by the scanner into the staging area
194 capture_surefire_reports(path)
195 capture_jacoco(path)
196 capture_sources(path)
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800197
Ray Milkey32963cf2018-11-21 09:31:51 -0800198
Ray Milkey8705cce2019-01-14 14:05:48 -0800199def _main():
200 global ONOS_ROOT, targets
201 debug = False
202 if len(sys.argv) > 1:
203 debug = True
Brian O'Connor9c2c8232016-11-08 17:13:14 -0800204
Ray Milkey8705cce2019-01-14 14:05:48 -0800205 # Change to $ONOS_ROOT
206 ONOS_ROOT = os.environ['ONOS_ROOT']
207 if ONOS_ROOT:
208 os.chdir(ONOS_ROOT)
209
210 # build ONOS
211 run_command(["bazel", "build", "onos"])
212
213 # find the test targets to get coverage for
214 if debug:
215 # Use a predefined list of targets for debugging
216 targets = ['//core/net:onos-core-net', '//utils/misc:onlab-misc']
217 else:
218 # Query all onos OSGi jar file rules with tests from bazel
219 targets = run_command(["bazel", "query", "attr('name', '.*-tests-gen', '//...')"])
220 targets = [target.replace("-tests-gen-deps", "") for target in targets]
221
222 # Filter out targets without any tests in them
223 targets_with_tests = []
224 for target in targets:
225 colon = target.find(':')
226 base_target = target[0:colon]
227 target_query_result = run_command(['bazel', 'query', 'tests(' + base_target + ':*)'])
228 for result_line in target_query_result:
229 if "src/test" in result_line:
230 targets_with_tests.append(target)
231 break
232 targets = targets_with_tests
233
234 # Clear out any old results
235 rmtree(SONAR_PROJECT, True)
236
237 # make a directory for the new results
238 make_dirs(SONAR_PROJECT)
239
240 # fill in the template for the sonar properties
241 sonar_parameters = {
Ray Milkey32963cf2018-11-21 09:31:51 -0800242 'name': 'onos',
243 'key': 'org.onosproject:onos',
244 'version': ONOS_VERSION,
Ray Milkey8705cce2019-01-14 14:05:48 -0800245 'jacoco': '/tmp/jacoco.exec',
246 'reports': 'surefire-reports',
247 'modules': ','.join([split_target(t)[1] for t in targets])
248 }
249 if debug:
250 sonar_parameters["key"] = 'org.onosproject:onos-test-sonar'
251 sonar_parameters["name"] = 'onos-test-sonar'
252
253 # Write the sonar properties file
254 with open(SONAR_PROPERTIES_FILE_NAME, 'w') as out:
255 out.write(ROOT_TEMPLATE % sonar_parameters)
256 for target in targets:
257 print "Processing coverage for target " + target
258 write_module(target, out)
259
260
261if __name__ == "__main__":
262 _main()