Merge "correct the way to check whether killed speaker1"
diff --git a/TestON/config/teston.cfg b/TestON/config/teston.cfg
index b255f45..aba93d1 100644
--- a/TestON/config/teston.cfg
+++ b/TestON/config/teston.cfg
@@ -5,6 +5,9 @@
         <class>xmlparser</class>
     </parser>
     <mail_to>hari@onlab.us</mail_to>
+    <mail_from></mail_from>
+    <mail_pass></mail_pass>
+    <mail_server></mail_server>
 
     <logger>
         <file>core/logger.py</file>
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index c65fa49..05693ba 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -120,6 +120,7 @@
         main.WikiFileName = main.logdir + "/" + main.TEST + "Wiki.txt"
         main.SummaryFileName = main.logdir + "/" + main.TEST + "Summary.txt"
         main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
+        main.TOTAL_TC_SUCCESS = 0
 
         #### Add log-level - Report
         logging.addLevelName(9, "REPORT")
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index bc23be0..4bbdc3c 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -97,7 +97,7 @@
         self.cleanupLock = threading.Lock()
         self.initiated = False
 
-        self.configparser()
+        self.config = self.configparser()
         verifyOptions( options )
         load_logger()
         self.componentDictionary = {}
@@ -742,11 +742,16 @@
         """
         Stop the test until Ctrl-D is entered.
         Ctrl-C will kill the test
+
+        Optional arguments:
+        email can either be a bool, or you can specify the email address
+        to send the email to
         """
         try:
             if email:
-                # FIXME: implement this
-                raise NotImplementedError
+                if '@' in email:
+                    main.mail = email
+                utilities.send_warning_email()
             self.log.error( "Test execution suspended. Press Ctrl-D to "
                             "resume or Ctrl-C to exit the test" )
             # NOTE: Ctrl-D needs to be entered on a new line
@@ -801,11 +806,20 @@
         main.logdir = main.FALSE
 
 def verifyMail( options ):
-    # Checking the mailing list
-    if options.mail:
+    # Mail-To: field
+    if options.mail:  # Test run specific
         main.mail = options.mail
-    else:
-        main.mail = main.params.get( 'mail', 'paxweb@paxterrasolutions.com' )
+    elif main.params.get('mail'):  # Test suite specific
+        main.mail = main.params.get( 'mail' )
+    else:  # TestON specific
+        main.mail = main.config['config'].get( 'mail_to' )
+    # Mail-From: field
+    main.sender = main.config['config'].get( 'mail_from' )
+    # Mail smtp server
+    main.smtp = main.config['config'].get( 'mail_server' )
+    # Mail-From account password
+    main.senderPwd = main.config['config'].get( 'mail_pass' )
+
 
 def verifyTestCases( options ):
     # Getting Test cases list
diff --git a/TestON/core/utilities.py b/TestON/core/utilities.py
index d5a5c0f..cdf6c1a 100644
--- a/TestON/core/utilities.py
+++ b/TestON/core/utilities.py
@@ -215,37 +215,67 @@
         msg = email.mime.Multipart.MIMEMultipart()
         try :
             if main.test_target:
-                sub = "Result summary of \""+main.TEST+"\" run on component \""+main.test_target+"\" Version \""+vars(main)[main.test_target].get_version()+"\": "+str(main.TOTAL_TC_SUCCESS)+"% Passed"
+                sub = "Result summary of \"" + main.TEST + "\" run on component \"" +\
+                      main.test_target + "\" Version \"" +\
+                      vars( main )[main.test_target].get_version() + "\": " +\
+                      str( main.TOTAL_TC_SUCCESS ) + "% Passed"
             else :
-                sub = "Result summary of \""+main.TEST+"\": "+str(main.TOTAL_TC_SUCCESS)+"% Passed"
+                sub = "Result summary of \"" + main.TEST + "\": " +\
+                      str( main.TOTAL_TC_SUCCESS ) + "% Passed"
         except ( KeyError, AttributeError ):
-            sub = "Result summary of \""+main.TEST+"\": "+str(main.TOTAL_TC_SUCCESS)+"% Passed"
+            sub = "Result summary of \"" + main.TEST + "\": " +\
+                  str( main.TOTAL_TC_SUCCESS ) + "% Passed"
 
         msg['Subject'] = sub
-        msg['From'] = 'paxweb@paxterrasolutions.com'
+        msg['From'] = main.sender
         msg['To'] = main.mail
-        #msg['Cc'] = 'paxweb@paxterrasolutions.com'
 
         # The main body is just another attachment
-        body = email.mime.Text.MIMEText(main.logHeader+"\n"+main.testResult)
-        msg.attach(body)
+        body = email.mime.Text.MIMEText( main.logHeader + "\n" +
+                                         main.testResult)
+        msg.attach( body )
 
-        # Attachment
-        for filename in os.listdir(main.logdir):
-            filepath = main.logdir+"/"+filename
-            fp=open(filepath,'rb')
-            att = email.mime.application.MIMEApplication(fp.read(),_subtype="")
+        # Attachments
+        for filename in os.listdir( main.logdir ):
+            filepath = main.logdir + "/" + filename
+            fp = open( filepath, 'rb' )
+            att = email.mime.application.MIMEApplication( fp.read(),
+                                                          _subtype="" )
             fp.close()
-            att.add_header('Content-Disposition','attachment',filename=filename)
-            msg.attach(att)
-
-        smtp = smtplib.SMTP('198.57.211.46')
-        smtp.starttls()
-        smtp.login('paxweb@paxterrasolutions.com','pax@peace')
-        smtp.sendmail(msg['From'],[msg['To']], msg.as_string())
-        smtp.quit()
+            att.add_header( 'Content-Disposition',
+                            'attachment',
+                            filename=filename )
+            msg.attach( att )
+        try:
+            smtp = smtplib.SMTP( main.smtp )
+            smtp.starttls()
+            smtp.login( main.sender, main.senderPwd )
+            smtp.sendmail( msg['From'], [msg['To']], msg.as_string() )
+            smtp.quit()
+        except Exception:
+            main.log.exception( "Error sending email" )
         return main.TRUE
 
+    def send_warning_email( self, subject=None ):
+        try:
+            if not subject:
+                subject = main.TEST + " PAUSED!"
+            # Create a text/plain message
+            msg = email.mime.Multipart.MIMEMultipart()
+
+            msg['Subject'] = subject
+            msg['From'] = main.sender
+            msg['To'] = main.mail
+
+            smtp = smtplib.SMTP( main.smtp )
+            smtp.starttls()
+            smtp.login( main.sender, main.senderPwd )
+            smtp.sendmail( msg['From'], [msg['To']], msg.as_string() )
+            smtp.quit()
+        except Exception:
+            main.log.exception( "" )
+            return main.FALSE
+        return main.TRUE
 
     def parse(self,fileName):
         '''
diff --git a/TestON/tests/CHOtest/CHOtest.params b/TestON/tests/CHOtest/CHOtest.params
index 965e297..4f34b11 100644
--- a/TestON/tests/CHOtest/CHOtest.params
+++ b/TestON/tests/CHOtest/CHOtest.params
@@ -18,7 +18,7 @@
     # 19X. IPv6 ping across Point,Multi-single,Single-Multi Intents
 
 <testcases>
-1,20,3,[40,5,140,60,160,70,170,80,180,10,5,90,190,71,171,81,181,10,5]*10,21,3,[41,5,141,61,161,72,172,82,182,10,5,91,191,73,173,83,183,10,5]*10,22,3,[42,5,142,62,162,74,174,84,184,10,5,92,192,75,175,85,185,10,5]*10
+1,20,3,[40,5,140,60,160,70,170,80,180,10,5,90,190,71,171,81,181,10,5]*50,21,3,[41,5,141,61,161,72,172,82,182,10,5,91,191,73,173,83,183,10,5]*50,22,3,[42,5,142,62,162,74,174,84,184,10,5,92,192,75,175,85,185,10,5]*50
 </testcases>
 
     <GIT>
@@ -94,7 +94,7 @@
     </timers>
 
     <TEST>
-        <pause_test>on</pause_test>
+        <pause_test>off</pause_test>
         <email>off</email>
         <intent_check>8</intent_check>
     </TEST>
diff --git a/TestON/tests/FUNCintent/Dependency/FuncIntentFunction.py b/TestON/tests/FUNCintent/Dependency/FuncIntentFunction.py
index 377fd20..9efe5d0 100644
--- a/TestON/tests/FUNCintent/Dependency/FuncIntentFunction.py
+++ b/TestON/tests/FUNCintent/Dependency/FuncIntentFunction.py
@@ -1468,6 +1468,7 @@
         main.log.info( itemName + ": Intents are installed correctly" )
     else:
         # Wait for at least 5 second before checking the intents again
+        main.log.error( "Intents are not installed correctly. Waiting 5 sec" )
         time.sleep( 5 )
         results = []
         # Second check of intents since some of the intents may be in
@@ -1478,6 +1479,7 @@
             results.append( tempResult )
         if all( result == main.TRUE for result in results ):
             main.log.info( itemName + ": Intents are installed correctly" )
+            intentResult = main.TRUE
         else:
             main.log.error( itemName + ": Intents are NOT installed correctly" )
             intentResult = main.FALSE
diff --git a/TestON/tests/FUNCintentRest/FUNCintentRest.params b/TestON/tests/FUNCintentRest/FUNCintentRest.params
index 4158f33..058b57a 100644
--- a/TestON/tests/FUNCintentRest/FUNCintentRest.params
+++ b/TestON/tests/FUNCintentRest/FUNCintentRest.params
@@ -3,7 +3,8 @@
     # 1 - Variable initialization and optional pull and build ONOS package
     # 2 - Install ONOS
     # 9 - Report logs
-    # 11 - Start Mininet
+    # 10 - Start Mininet with Openflow 1.0
+    # 11 - Start Mininet with Openflow 1.3
     # 12 - Assign switch to controller
     # 13 - Create a data of hosts information
     # 14 - Stop Mininet
@@ -11,11 +12,12 @@
     # 2000 - Test point intents
     # 3000 - Test single to multi point intents
     # 4000 - Test multi to single point intents
+    # 5000 - Test host mobility
 
-    <testcases>1,2,10,12,13,1000,2000</testcases>
+    <testcases>1,[2,10,12,13,1000,2000,14],[2,11,12,13,1000,2000,14]</testcases>
 
     <SCALE>
-        <size>1,3</size>
+        <size>1,1</size>
     </SCALE>
 
     <DEPENDENCY>
diff --git a/TestON/tests/FUNCovsdbtest/FUNCovsdbtest.py b/TestON/tests/FUNCovsdbtest/FUNCovsdbtest.py
index 95e90fd..3c76596 100644
--- a/TestON/tests/FUNCovsdbtest/FUNCovsdbtest.py
+++ b/TestON/tests/FUNCovsdbtest/FUNCovsdbtest.py
@@ -384,7 +384,8 @@
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="ovsdb node 1 manager is " + str( response ) ,
-                                 onfail="ovsdb node 1 manager check failed" )
+                                 onfail="ovsdb node 1 manager check failed\n" +\
+                                 str( main.OVSDB1.show() ) )
 
         main.step( "Check ovsdb node 2 manager is " + str( ctrlip ) )
         response = main.OVSDB2.getManager()
@@ -395,7 +396,8 @@
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="ovsdb node 2 manager is " + str( response ) ,
-                                 onfail="ovsdb node 2 manager check failed" )
+                                 onfail="ovsdb node 2 manager check failed\n" +\
+                                 str( main.OVSDB2.show() ) )
 
         main.step( "Check ovsdb node 1 bridge br-int controller set to " + str( ctrlip ) )
         response = main.OVSDB1.getController( "br-int" )
@@ -408,7 +410,7 @@
                                  onpass="Check ovsdb node 1 bridge br-int controller set to " +\
                                   str( ctrlip ) + " sucess",
                                  onfail="Check ovsdb node 1 bridge br-int controller set to " +\
-                                  str( ctrlip ) + " failed" )
+                                  str( ctrlip ) + " failed\n" + str( main.OVSDB1.show() ) )
 
         main.step( "Check ovsdb node 2 bridge br-int controller set to  " + str( ctrlip ) )
         response = main.OVSDB2.getController( "br-int" )
@@ -421,7 +423,7 @@
                                  onpass="Check ovsdb node 2 bridge br-int controller set to " +\
                                   str( ctrlip ) + " sucess",
                                  onfail="Check ovsdb node 2 bridge br-int controller set to " +\
-                                  str( ctrlip ) + " failed" )
+                                  str( ctrlip ) + " failed\n" + str( main.OVSDB2.show()) )
 
         main.step( "Check onoscli devices have ovs " + str( OVSDB1Ip ) )
         response = main.ONOScli1.devices()
diff --git a/TestON/tests/SCPFintentRerouteLat/SCPFintentRerouteLat.py b/TestON/tests/SCPFintentRerouteLat/SCPFintentRerouteLat.py
index 3b74dcf..0dafc51 100644
--- a/TestON/tests/SCPFintentRerouteLat/SCPFintentRerouteLat.py
+++ b/TestON/tests/SCPFintentRerouteLat/SCPFintentRerouteLat.py
@@ -170,7 +170,6 @@
         #from scipy import stats
 
         ts = time.time()
-        date = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d')
 
         sampleSize = int(main.params[ 'TEST' ][ 'sampleSize' ])
         warmUp = int(main.params[ 'TEST' ][ 'warmUp' ])
@@ -258,13 +257,24 @@
                         break
                     time.sleep(2)
 
+                if debug: main.log.debug("raw: " + raw)
+
                 temp = raw.splitlines()
-                for line in temp:
-                    if str(date) in line:
-                        temp = line
-                        break
+
+                if debug: main.log.debug("temp (after splitlines): " + str(temp))
+
+                # Since the string is deterministic the date is always the 3rd element.
+                # However, if the data were grepping for in the onos log changes then this will
+                # not work. This is why we print out the raw and temp string so we can visually
+                # check if everything is in the correct order. temp should like this:
+                # temp = ['/onos$ onos-ssh $OC1 cat /opt/onos/log/karaf.log | grep Top ', 
+                #         'ologyManager| tail -1', '2015-10-15 12:03:33,736 ... ]
+                temp = temp[2]
+
+                if debug: main.log.debug("temp (checking for date): " + str(temp))
 
                 cutTimestamp = (temp.split(" "))[0] + " " + (temp.split(" "))[1]
+
                 if debug: main.log.info("Cut timestamp: " + cutTimestamp)
 
                 #validate link count and flow count