blob: 78c3fe098390e43a4f31e17f95e6b938be1edc08 [file] [log] [blame]
Jeremy Ronquillodae11042018-02-21 09:21:44 -08001# Copyright 2018 Open Networking Foundation (ONF)
2#
3# Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
4# the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
5# or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
6#
7# TestON is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 2 of the License, or
10# (at your option) any later version.
11#
12# TestON is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with TestON. If not, see <http://www.gnu.org/licenses/>.
19#
20# Example script:
21# ALL tests (https://jenkins.onosproject.org/view/QA/job/postjob-VM/lastSuccessfulBuild/artifact/ALL_master_build-latest_test-suite-summary.jpg):
22# Rscript histogramMultipleTestGroups.R <url> <port> <username> <pass> ALL master "FUNCbgpls,FUNCflow,FUNCformCluster,FUNCgroup,FUNCintent,FUNCintentRest,FUNCipv6Intent,FUNCnetCfg,FUNCnetconf,FUNCoptical,FUNCovsdbtest,FUNCvirNetNB,HAbackupRecover,HAclusterRestart,HAfullNetPartition,HAkillNodes,HAsanity,HAscaling,HAsingleInstanceRestart,HAstopNodes,HAswapNodes,HAupgrade,HAupgradeRollback,PLATdockertest,USECASE_SdnipFunction,USECASE_SdnipFunctionCluster,VPLSBasic,VPLSfailsafe" latest /path/to/save/directory/
23
24# **********************************************************
25# STEP 1: Data management.
26# **********************************************************
27
28print( "**********************************************************" )
29print( "STEP 1: Data management." )
30print( "**********************************************************" )
31
32# Command line arguments are read. Args include the database credentials, test name, branch name, and the directory to output files.
33print( "Reading commmand-line args." )
34args <- commandArgs( trailingOnly=TRUE )
35
36testsToInclude <- 7
37build_to_show <- 8
38save_directory <- 9
39
40# ----------------
41# Import Libraries
42# ----------------
43
44print( "Importing libraries." )
45library( ggplot2 )
46library( reshape2 )
47library( RPostgreSQL )
Devin Lim324806b2018-05-11 15:36:52 -070048source( "~/OnosSystemTest/TestON/JenkinsFile/wikiGraphRScripts/dependencies/saveGraph.R" )
49source( "~/OnosSystemTest/TestON/JenkinsFile/wikiGraphRScripts/dependencies/fundamentalGraphData.R" )
50source( "~/OnosSystemTest/TestON/JenkinsFile/wikiGraphRScripts/dependencies/initSQL.R" )
51source( "~/OnosSystemTest/TestON/JenkinsFile/wikiGraphRScripts/dependencies/cliArgs.R" )
Jeremy Ronquillodae11042018-02-21 09:21:44 -080052
53# -------------------
54# Check CLI Arguments
55# -------------------
56
57print( "Verifying CLI args." )
58
59if ( length( args ) != save_directory ){
60 specialArgs <- c( "tests-to-include-(as-one-string-sep-groups-by-semicolon-title-as-first-group-item-sep-by-dash)",
61 "build-to-show" )
62 usage( "histogramMultipleTestGroups.R", specialArgs )
63 quit( status = 1 )
64}
65
66# ------------------
67# SQL Initialization
68# ------------------
69
70print( "Initializing SQL" )
71
72con <- initSQL( args[ database_host ],
73 args[ database_port ],
74 args[ database_u_id ],
75 args[ database_pw ] )
76
77# ---------------------
78# Test Case SQL Command
79# ---------------------
80
81print( "Generating Test Case SQL command." )
82
83sqlCommands <- generateGroupedTestSingleBuildSQLCommand( args[ branch_name ],
84 args[ testsToInclude ],
85 args[ build_to_show ] )
86
87titles <- getTitlesFromGroupTest( args[ branch_name ],
88 args[ testsToInclude ] )
89
90dbResults <- list()
91i <- 1
92for ( command in sqlCommands ){
93 dbResults[[i]] <- retrieveData( con, command )
94 i <- i + 1
95}
96
97print( "dbResult:" )
98print( dbResults )
99
100# -------------------------------
101# Create Title and Graph Filename
102# -------------------------------
103
104print( "Creating title of graph." )
105
106titlePrefix <- paste( args[ graph_title ], " ", sep="" )
107if ( args[ graph_title ] == "ALL" ){
108 titlePrefix <- ""
109}
110
111if ( args[ build_to_show ] == "latest" ){
112 buildTitle <- "\nLatest Test Results"
113 filebuild_to_show <- "latest"
114} else {
115 buildTitle <- paste( " \n Build #", args[ build_to_show ], sep="" )
116 filebuild_to_show <- args[ build_to_show ]
117}
118
119title <- paste( titlePrefix,
120 "Summary of Test Suites - ",
121 args[ branch_name ],
122 buildTitle,
123 sep="" )
124
125print( "Creating graph filename." )
126
127outputFile <- paste( args[ save_directory ],
128 args[ graph_title ],
129 "_",
130 args[ branch_name ],
131 "_build-",
132 filebuild_to_show,
133 "_test-suite-summary.jpg",
134 sep="" )
135
136# **********************************************************
137# STEP 2: Organize data.
138# **********************************************************
139
140print( "**********************************************************" )
141print( "STEP 2: Organize Data." )
142print( "**********************************************************" )
143
144passNum <- list()
145failNum <- list()
146exeNum <- list()
147skipNum <- list()
148totalNum <- list()
149
150passPercent <- list()
151failPercent <- list()
152exePercent <- list()
153nonExePercent <- list()
154
155actualPassPercent <- list()
156actualFailPercent <- list()
157
158appName <- c()
159afpName <- c()
160nepName <- c()
161
162tmpPos <- c()
163tmpCases <- c()
164
165for ( i in 1:length( dbResults ) ){
166 t <- dbResults[[i]]
167
168 passNum[[i]] <- sum( t$num_passed )
169 failNum[[i]] <- sum( t$num_failed )
170 exeNum[[i]] <- passNum[[i]] + failNum[[i]]
171 totalNum[[i]] <- sum( t$num_planned )
172 skipNum[[i]] <- totalNum[[i]] - exeNum[[i]]
173
174 passPercent[[i]] <- passNum[[i]] / exeNum[[i]]
175 failPercent[[i]] <- failNum[[i]] / exeNum[[i]]
176 exePercent[[i]] <- exeNum[[i]] / totalNum[[i]]
177 nonExePercent[[i]] <- ( 1 - exePercent[[i]] ) * 100
178
179 actualPassPercent[[i]] <- passPercent[[i]] * exePercent[[i]] * 100
180 actualFailPercent[[i]] <- failPercent[[i]] * exePercent[[i]] * 100
181
182 appName <- c( appName, "Passed" )
183 afpName <- c( afpName, "Failed" )
184 nepName <- c( nepName, "Skipped/Unexecuted" )
185
186 tmpPos <- c( tmpPos, 100 - ( nonExePercent[[i]] / 2 ), actualPassPercent[[i]] + actualFailPercent[[i]] - ( actualFailPercent[[i]] / 2 ), actualPassPercent[[i]] - ( actualPassPercent[[i]] / 2 ) )
187 tmpCases <- c( tmpCases, skipNum[[i]], failNum[[i]], passNum[[i]] )
188}
189
190relativePosLength <- length( dbResults ) * 3
191
192relativePos <- c()
193relativeCases <- c()
194
195for ( i in 1:3 ){
196 relativePos <- c( relativePos, tmpPos[ seq( i, relativePosLength, 3 ) ] )
197 relativeCases <- c( relativeCases, tmpCases[ seq( i, relativePosLength, 3 ) ] )
198}
199names( actualPassPercent ) <- appName
200names( actualFailPercent ) <- afpName
201names( nonExePercent ) <- nepName
202
203labels <- paste( titles, "\n", totalNum, " Test Cases", sep="" )
204
205# --------------------
206# Construct Data Frame
207# --------------------
208
209print( "Constructing Data Frame" )
210
211dataFrame <- melt( c( nonExePercent, actualFailPercent, actualPassPercent ) )
212dataFrame$title <- seq( 1, length( dbResults ), by = 1 )
213colnames( dataFrame ) <- c( "perc", "key", "suite" )
214
215dataFrame$xtitles <- labels
216dataFrame$relativePos <- relativePos
217dataFrame$relativeCases <- relativeCases
218dataFrame$valueDisplay <- c( paste( round( dataFrame$perc, digits = 2 ), "% - ", relativeCases, " Tests", sep="" ) )
219
220dataFrame$key <- factor( dataFrame$key, levels=unique( dataFrame$key ) )
221
222dataFrame$willDisplayValue <- dataFrame$perc > 15.0 / length( dbResults )
223
224for ( i in 1:nrow( dataFrame ) ){
225 if ( relativeCases[[i]] == "1" ){
226 dataFrame[ i, "valueDisplay" ] <- c( paste( round( dataFrame$perc[[i]], digits = 2 ), "% - ", relativeCases[[i]], " Test", sep="" ) )
227 }
228 if ( !dataFrame[ i, "willDisplayValue" ] ){
229 dataFrame[ i, "valueDisplay" ] <- ""
230 }
231}
232
233print( "Data Frame Results:" )
234print( dataFrame )
235
236# **********************************************************
237# STEP 3: Generate graphs.
238# **********************************************************
239
240print( "**********************************************************" )
241print( "STEP 3: Generate Graph." )
242print( "**********************************************************" )
243
244# -------------------
245# Main Plot Generated
246# -------------------
247
248print( "Creating main plot." )
249
250mainPlot <- ggplot( data = dataFrame, aes( x = suite,
251 y = perc,
252 fill = key ) )
253
254# ------------------------------
255# Fundamental Variables Assigned
256# ------------------------------
257
258print( "Generating fundamental graph data." )
259
260theme_set( theme_grey( base_size = 26 ) ) # set the default text size of the graph.
261
262xScaleConfig <- scale_x_continuous( breaks = dataFrame$suite,
263 label = dataFrame$xtitles )
264yScaleConfig <- scale_y_continuous( breaks = seq( 0, 100,
265 by = 10 ) )
266
267xLabel <- xlab( "" )
268yLabel <- ylab( "Total Test Cases (%)" )
269
270theme <- graphTheme() + theme( axis.text.x = element_text( angle = 0, size = 25 - 1.25 * length( dbResults ) ) )
271
272title <- labs( title = title, subtitle = lastUpdatedLabel() )
273
274# Store plot configurations as 1 variable
275fundamentalGraphData <- mainPlot +
276 xScaleConfig +
277 yScaleConfig +
278 xLabel +
279 yLabel +
280 theme +
281 title
282
283# ---------------------------
284# Generating Bar Graph Format
285# ---------------------------
286
287print( "Generating bar graph." )
288
289unexecutedColor <- webColor( "gray" ) # Gray
290failedColor <- webColor( "red" ) # Red
291passedColor <- webColor( "green" ) # Green
292
293colors <- scale_fill_manual( values=c( if ( "Skipped/Unexecuted" %in% dataFrame$key ){ unexecutedColor },
294 if ( "Failed" %in% dataFrame$key ){ failedColor },
295 if ( "Passed" %in% dataFrame$key ){ passedColor } ) )
296
297barGraphFormat <- geom_bar( stat = "identity", width = 0.8 )
298
299barGraphValues <- geom_text( aes( x = dataFrame$suite,
300 y = dataFrame$relativePos,
301 label = format( paste( dataFrame$valueDisplay ) ) ),
302 size = 15.50 / length( dbResults ) + 2.33, fontface = "bold" )
303
304result <- fundamentalGraphData +
305 colors +
306 barGraphFormat +
307 barGraphValues
308
309# -----------------------
310# Exporting Graph to File
311# -----------------------
312
313saveGraph( outputFile ) # from saveGraph.R