[ONOS-7341]: Statistic Graphs Representing All System Tests
Change-Id: Ibd8a30a709c791dfc5f01b41b7cefe37ed3348e7
diff --git a/TestON/JenkinsFile/scripts/testCategoryBuildStats.R b/TestON/JenkinsFile/scripts/testCategoryBuildStats.R
index 0b4d4b5..94c3572 100644
--- a/TestON/JenkinsFile/scripts/testCategoryBuildStats.R
+++ b/TestON/JenkinsFile/scripts/testCategoryBuildStats.R
@@ -1,4 +1,4 @@
-# Copyright 2017 Open Networking Foundation (ONF)
+# Copyright 2018 Open Networking Foundation (ONF)
#
# Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
# the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
@@ -40,9 +40,7 @@
branchName <- 6
testsToInclude <- 7
buildToShow <- 8
-displayStatus <- 9
-scaleOfPercent <- 10
-saveDirectory <- 11
+saveDirectory <- 9
# ----------------
# Import Libraries
@@ -61,17 +59,15 @@
if ( is.na( args[ saveDirectory ] ) ){
- print( paste( "Usage: Rscript testCategoryTrend.R",
+ print( paste( "Usage: Rscript testCategoryBuildStats.R",
"<database-host>",
"<database-port>",
"<database-user-id>",
"<database-password>",
"<test-suite-name>",
"<branch-name>",
- "<tests-to-include-(as-one-string)>",
+ "<tests-to-include-(as-one-string-sep-groups-by-semicolon-title-as-first-group-item-sep-by-dash)>",
"<build-to-show>",
- "<pass/fail/plan>",
- "<percent-scale>",
"<directory-to-save-graphs>",
sep=" " ) )
@@ -94,6 +90,7 @@
# ---------------------
# Test Case SQL Command
# ---------------------
+
print( "Generating Test Case SQL command." )
tests <- "'"
@@ -104,37 +101,56 @@
fileBuildToShow <- args[ buildToShow ]
operator <- "= "
+buildTitle <- ""
if ( args[ buildToShow ] == "latest" ){
+ buildTitle <- "\nLatest Test Results"
operator <- ">= "
args[ buildToShow ] <- "1000"
+} else {
+ buildTitle <- paste( " \n Build #", args[ buildToShow ] , sep="" )
}
-command <- paste( "SELECT * ",
- "FROM executed_test_tests a ",
- "WHERE ( SELECT COUNT( * ) FROM executed_test_tests b ",
- "WHERE b.branch='",
- args[ branchName ],
- "' AND b.actual_test_name IN (",
- tests,
- ") AND a.actual_test_name = b.actual_test_name AND a.date <= b.date AND b.build ", operator,
- args[ buildToShow ],
- " ) = ",
- 1,
- " AND a.branch='",
- args[ branchName ],
- "' AND a.actual_test_name IN (",
- tests,
- ") AND a.build ", operator,
- args[ buildToShow ],
- " ORDER BY a.actual_test_name DESC, a.date DESC",
- sep="")
+tests <- strsplit( args[ testsToInclude ], ";" )
+dbResults <- list()
+titles <- list()
-print( "Sending SQL command:" )
-print( command )
-dbResult <- dbGetQuery( con, command )
+for ( i in 1:length( tests[[1]] ) ){
+ splitTestList <- strsplit( tests[[1]][ i ], "-" )
+ testList <- splitTestList[[1]][2]
+ titles[[i]] <- splitTestList[[1]][1]
-maxBuild <- max( dbResult[ 'build' ] )
-dbResult <- dbResult[ which( dbResult[,4]>=maxBuild ), ]
+ testsCommand <- "'"
+ for ( test in as.list( strsplit( testList, "," )[[1]] ) ){
+ testsCommand <- paste( testsCommand, test, "','", sep="" )
+ }
+ testsCommand <- substr( testsCommand, 0, nchar( testsCommand ) - 2 )
+
+ command <- paste( "SELECT * ",
+ "FROM executed_test_tests a ",
+ "WHERE ( SELECT COUNT( * ) FROM executed_test_tests b ",
+ "WHERE b.branch='",
+ args[ branchName ],
+ "' AND b.actual_test_name IN (",
+ testsCommand,
+ ") AND a.actual_test_name = b.actual_test_name AND a.date <= b.date AND b.build ", operator,
+ args[ buildToShow ],
+ " ) = ",
+ 1,
+ " AND a.branch='",
+ args[ branchName ],
+ "' AND a.actual_test_name IN (",
+ testsCommand,
+ ") AND a.build ", operator,
+ args[ buildToShow ],
+ " ORDER BY a.actual_test_name DESC, a.date DESC",
+ sep="")
+ print( "Sending SQL command:" )
+ print( command )
+ dbResults[[i]] <- dbGetQuery( con, command )
+}
+
+print( "dbResult:" )
+print( dbResults )
# -------------------------------
# Create Title and Graph Filename
@@ -142,25 +158,15 @@
print( "Creating title of graph." )
-titleDisplayStatus <- ""
-if ( args[ displayStatus ] == "fail" ){
- titleDisplayStatus <- "Failed"
-} else if ( args[ displayStatus ] == "plan" ){
- titleDisplayStatus <- "Executed"
-} else if ( args[ displayStatus ] == "pass" ){
- titleDisplayStatus <- "Succeeded"
-} else {
- print( paste( "[ERROR]: Invalid histogram display status: ", args[ displayStatus ], sep="" ) )
- quit( status = 1 )
+titlePrefix <- paste( args[ testSuiteName ], " ", sep="" )
+if ( args[ testSuiteName ] == "ALL" ){
+ titlePrefix <- ""
}
-title <- paste( args[ testSuiteName ],
- " Tests ",
- titleDisplayStatus,
- " - ",
+title <- paste( titlePrefix,
+ "Summary of Test Suites - ",
args[ branchName ],
- " \n Build #",
- max( dbResult[ 'build' ] ),
+ buildTitle,
sep="" )
print( "Creating graph filename." )
@@ -171,16 +177,9 @@
args[ branchName ],
"_build-",
fileBuildToShow,
- "_",
- args[ scaleOfPercent ],
- "-scaling",
- "_",
- args[ displayStatus ],
- "_histogram.jpg",
+ "_test-suite-summary.jpg",
sep="" )
-print( dbResult )
-
# **********************************************************
# STEP 2: Organize data.
# **********************************************************
@@ -189,23 +188,94 @@
print( "STEP 2: Organize Data." )
print( "**********************************************************" )
-t <- subset( dbResult, select=c( "actual_test_name", "num_passed", "num_failed", "num_planned" ) )
-t$passed_percent <- t$num_passed / t$num_planned * 100
-t$failed_percent <- t$num_failed / t$num_planned * 100
-t$planned_percent <- ( t$num_passed + t$num_failed ) / t$num_planned * 100
+passNum <- list()
+failNum <- list()
+exeNum <- list()
+skipNum <- list()
+totalNum <- list()
+
+passPercent <- list()
+failPercent <- list()
+exePercent <- list()
+nonExePercent <- list()
+
+actualPassPercent <- list()
+actualFailPercent <- list()
+
+appName <- c()
+afpName <- c()
+nepName <- c()
+
+tmpPos <- c()
+tmpCases <- c()
+
+for ( i in 1:length( dbResults ) ){
+ t <- dbResults[[i]]
+
+ passNum[[i]] <- sum( t$num_passed )
+ failNum[[i]] <- sum( t$num_failed )
+ exeNum[[i]] <- passNum[[i]] + failNum[[i]]
+ totalNum[[i]] <- sum( t$num_planned )
+ skipNum[[i]] <- totalNum[[i]] - exeNum[[i]]
+
+ passPercent[[i]] <- passNum[[i]] / exeNum[[i]]
+ failPercent[[i]] <- failNum[[i]] / exeNum[[i]]
+ exePercent[[i]] <- exeNum[[i]] / totalNum[[i]]
+ nonExePercent[[i]] <- ( 1 - exePercent[[i]] ) * 100
+
+ actualPassPercent[[i]] <- passPercent[[i]] * exePercent[[i]] * 100
+ actualFailPercent[[i]] <- failPercent[[i]] * exePercent[[i]] * 100
+
+ appName <- c( appName, "Passed" )
+ afpName <- c( afpName, "Failed" )
+ nepName <- c( nepName, "Skipped/Unexecuted" )
+
+ tmpPos <- c( tmpPos, 100 - ( nonExePercent[[i]] / 2 ), actualPassPercent[[i]] + actualFailPercent[[i]] - ( actualFailPercent[[i]] / 2 ), actualPassPercent[[i]] - ( actualPassPercent[[i]] / 2 ) )
+ tmpCases <- c( tmpCases, skipNum[[i]], failNum[[i]], passNum[[i]] )
+}
+
+relativePosLength <- length( dbResults ) * 3
+
+relativePos <- c()
+relativeCases <- c()
+
+for ( i in 1:3 ){
+ relativePos <- c( relativePos, tmpPos[ seq( i, relativePosLength, 3 ) ] )
+ relativeCases <- c( relativeCases, tmpCases[ seq( i, relativePosLength, 3 ) ] )
+}
+names( actualPassPercent ) <- appName
+names( actualFailPercent ) <- afpName
+names( nonExePercent ) <- nepName
+
+labels <- paste( titles, "\n", totalNum, " Test Cases", sep="" )
# --------------------
# Construct Data Frame
# --------------------
-dataFrame <- aggregate( t$passed_percent, by=list( Category=t$actual_test_name ), FUN=sum )
-if ( args[ displayStatus ] == "fail" ){
- dataFrame <- aggregate( t$failed_percent, by=list( Category=t$actual_test_name ), FUN=sum )
-} else if ( args[ displayStatus ] == "plan" ){
- dataFrame <- aggregate( t$planned_percent, by=list( Category=t$actual_test_name ), FUN=sum )
-}
+print( "Constructing Data Frame" )
-colnames( dataFrame ) <- c( "Test", paste( titleDisplayStatus, "%", sep="" ) )
+dataFrame <- melt( c( nonExePercent, actualFailPercent, actualPassPercent ) )
+dataFrame$title <- seq( 1, length( dbResults ), by = 1 )
+colnames( dataFrame ) <- c( "perc", "key", "suite" )
+
+dataFrame$xtitles <- labels
+dataFrame$relativePos <- relativePos
+dataFrame$relativeCases <- relativeCases
+dataFrame$valueDisplay <- c( paste( round( dataFrame$perc, digits = 2 ), "% - ", relativeCases, " Tests", sep="" ) )
+
+dataFrame$key <- factor( dataFrame$key, levels=unique( dataFrame$key ) )
+
+dataFrame$willDisplayValue <- dataFrame$perc > 15.0 / length( dbResults )
+
+for ( i in 1:nrow( dataFrame ) ){
+ if ( relativeCases[[i]] == "1" ){
+ dataFrame[ i, "valueDisplay" ] <- c( paste( round( dataFrame$perc[[i]], digits = 2 ), "% - ", relativeCases[[i]], " Test", sep="" ) )
+ }
+ if ( !dataFrame[ i, "willDisplayValue" ] ){
+ dataFrame[ i, "valueDisplay" ] <- ""
+ }
+}
print( "Data Frame Results:" )
print( dataFrame )
@@ -231,13 +301,14 @@
# - y: y-axis values (usually tests)
# - color: the category of the colored lines (usually status of test)
-mainPlot <- ggplot( data = dataFrame, aes( dataFrame[ ,2 ] ) )
-
# -------------------
# Main Plot Formatted
# -------------------
print( "Formatting main plot." )
+mainPlot <- ggplot( data = dataFrame, aes( x = suite,
+ y = perc,
+ fill = key ) )
# ------------------------------
# Fundamental Variables Assigned
@@ -247,11 +318,13 @@
theme_set( theme_grey( base_size = 26 ) ) # set the default text size of the graph.
-xScaleConfig <- scale_x_continuous( breaks = seq( 0, 100, by = 10 ) )
-yScaleConfig <- scale_y_continuous( breaks = seq( 0, nrow( dbResult ), by = 1 ), limits = c( 0, nrow( dbResult ) ) )
+xScaleConfig <- scale_x_continuous( breaks = dataFrame$suite,
+ label = dataFrame$xtitles )
+yScaleConfig <- scale_y_continuous( breaks = seq( 0, 100,
+ by = 10 ) )
-xLabel <- xlab( paste( titleDisplayStatus, "%" ) )
-yLabel <- ylab( "Frequency" )
+xLabel <- xlab( "" )
+yLabel <- ylab( "Total Test Cases (%)" )
imageWidth <- 15
imageHeight <- 10
@@ -259,7 +332,7 @@
# Set other graph configurations here.
theme <- theme( plot.title = element_text( hjust = 0.5, size = 32, face ='bold' ),
- axis.text.x = element_text( angle = 0, size = 14 ),
+ axis.text.x = element_text( angle = 0, size = 25 - 1.25 * length( dbResults ) ),
legend.position = "bottom",
legend.text = element_text( size = 22 ),
legend.title = element_blank(),
@@ -279,26 +352,31 @@
theme +
title
-# ----------------------------
-# Generating Line Graph Format
-# ----------------------------
+# ---------------------------
+# Generating Bar Graph Format
+# ---------------------------
-print( "Generating line graph." )
+print( "Generating bar graph." )
-barColor <- "#00B208"
-if ( args[ displayStatus ] == "fail" ){
- barColor <- "#E80000"
-} else if ( args[ displayStatus ] == "plan" ){
- barColor <- "#00A5FF"
-}
+unexecutedColor <- "#CCCCCC" # Gray
+failedColor <- "#E02020" # Red
+passedColor <- "#16B645" # Green
-histogramFormat <- geom_histogram( col = "#000000",
- fill = barColor,
- breaks = seq( 0, 100, by = strtoi( args[ scaleOfPercent ] ) ),
- lwd = 0.5 )
+colors <- scale_fill_manual( values=c( if ( "Skipped/Unexecuted" %in% dataFrame$key ){ unexecutedColor },
+ if ( "Failed" %in% dataFrame$key ){ failedColor },
+ if ( "Passed" %in% dataFrame$key ){ passedColor } ) )
+
+barGraphFormat <- geom_bar( stat = "identity", width = 0.8 )
+
+barGraphValues <- geom_text( aes( x = dataFrame$suite,
+ y = dataFrame$relativePos,
+ label = format( paste( dataFrame$valueDisplay ) ) ),
+ size = 15.50 / length( dbResults ) + 2.33, fontface = "bold" )
result <- fundamentalGraphData +
- histogramFormat
+ colors +
+ barGraphFormat +
+ barGraphValues
# -----------------------
# Exporting Graph to File