CDash:Dart2AndCDashSubmission

< CDash Main Page

Setting up dual Dart2 and CDash Submissions

If you are moving from Dart2 to CDash you may want to submit to both dashboards in parallel for a while, before switching solely to CDash.

The following shows how to setup CTest to use a trigger script, which in turn does a submit to both Dart2 (via XML-RPC) and CDash (via HTTP).

Note: This method doesn't support submission of Dart2 notes files.

Configure CTest to use a trigger script:

SET (CTEST_DROP_METHOD "http")
SET (CTEST_DROP_SITE "mysite.org")
SET (CTEST_DROP_LOCATION "/cgi-bin/HTTPUploadDartFile.cgi")
SET (CTEST_TRIGGER_SITE "http://${DROP_SITE}/cgi-bin/TriggerSiteDart2AndCDash.cgi")

Both HTTPUploadDartFile.cgi and TriggerSiteDart2AndCDash.cgi have a hard coded path that may require editing: /srv/dart2/incoming

This current implementation only supports a single hard coded project in TriggerSiteDart2AndCDash.cgi. You can of course create a specific trigger site per project to workaround this.

HTTPUploadDartFile.cgi

#!/usr/bin/env python
#
# Script that will upload files to specified directory on the server, where
# triggering script will find it.
#
# Installation:
#    Place this script in your cgi-bin area.
#    Change the variable "incoming_directory" to match your
#        installation.
###

import cgi
import re
import sys
import os

print "Content-type: text/html"
print

done = 0

incoming_directory = '/srv/dart2/incoming'

form = cgi.FieldStorage()

# Debug
#try:
#  tmpfile = open("/tmp/http_upload_log_file.log", "w")
#  tmpfile.write(`form`)
#  tmpfile.close()
#except Exception: pass

FileData = ""
FileName = ""

# process form
if type(form.value) == type("foo"):
 if "QUERY_STRING" in os.environ:
   dt = cgi.parse_qs(os.environ["QUERY_STRING"])
   if "FileName" in dt:
     FileName = dt["FileName"][0]
   FileData = form.value
elif "FileName" in form:
 FileName = form["FileName"].value
 if "FileData" in form:
    FileData = "form"

# verify file specified
if not FileData or not FileName:
  print "Please send file. "
  print "FileName has to contain file name"
  print "FileData has to contain file content"
  print form
  sys.exit(0)

print "Received file: " + FileName

# check if valid file name
filename = re.sub("[/\\|:%%]", "_", FileName)
if re.match(".+___.+___[0-9]{8}-[0-9]{4}-(Experimental|Nightly|Continuous)___XML___(Update|Configure|Build|Test|Coverage|Purify).xml", filename):
  print "Correct file name"
else:
  print "Can only upload files with format:"
  print "<site>___<BuildType>___<TAG>-<TestType>___XML___<File>.xml"
  sys.exit(0)

# get the file data
file_lines = ""
if FileData == "form":
  fileitem = form["FileData"]
  if fileitem.file:
    file_lines = fileitem.file.read()
  else:
    print "Not a file"
else:
  file_lines = FileData

# write to a file on disk
if file_lines:
  file = open(incoming_directory + "/" + filename, "w")
  if not file:
    print "Cannot create file: %s/%s" % ( incoming_directory, filename )
    sys.exit(0)
  file.write(file_lines)
  file.close()
  print "Thank you for the file"
  done = 1

if not done:
  print "Problem summiting data"

TriggerSiteDart2AndCDash.cgi

Note: If the script fails it will create log files in /tmp/tmp*.html.

#!/usr/bin/env python
#
# Script that will resubmit incoming testing results to
# Dart2 (via XMLRPC) and CDash (via HTTP).
#
# This scripts intended use is to allow the transition from
# Dart2 to CDash, by allowing both to be run in parallel.
#
###

import cgi
import cgitb; cgitb.enable(display=0, logdir="/tmp")

import sys
import os
import xmlrpclib
import put

def Trigger(dart2Url, cdashUrl, projectName, dropLocation):
	print "Content-Type: text/html"
	print

	form = cgi.FieldStorage()
	xmlfile = ""
	if "xmlfile" in form:
		xmlfile = form["xmlfile"].value

	if not xmlfile:
		print "No submission file"
		sys.exit(1)

	ipfile = dropLocation + xmlfile

	try:
		dart2Server = xmlrpclib.ServerProxy(dart2Url + projectName + "/Command/")

		try:
			fp = open(ipfile)
			content = fp.read()
			bin = xmlrpclib.Binary(content)
			print "Server responded: [%s]" % dart2Server.Submit.put(bin)
			fp.close()
		except Exception, v:
			print "ERROR", v
			print "Unexpected error:", sys.exc_info()
	except:
		print "Problem submitting XML-RPC for the file: %s" % xmlfile

	try:
		cdashServer = cdashUrl + "CDash/submit.php?project=" + projectName

		f = open(ipfile, 'rb')
		put.putfile(f, cdashServer)
		f.close()
	except:
		print "Problem submitting HTTP for the file: %s" % xmlfile

	os.unlink(ipfile)


if __name__ == "__main__":
	# Ensure all paths have a trailing slash
	# Dart2 URL, CDash URL, project, incoming directory
	Trigger("http://localhost:8081/", "http://localhost/", "MyProject", "/srv/dart2/incoming/")

put.py

Note: This version of put requires Apache2 (for chunking).

#!/usr/bin/env python
"""
put.py - Python HTTP PUT Client
Author: Sean B. Palmer, inamidst.com

Basic API usage, once with optional auth:

import put
put.putname('test.txt', 'http://example.org/test')

f = open('test.txt', 'rb')
put.putfile(f, 'http://example.org/test')
f.close()

bytes = open('test.txt', 'rb').read()
auth = {'username': 'myuser', 'password': 'mypass'}
put.put(bytes, 'http://example.org/test', **auth)
"""

import sys, httplib, urlparse
from optparse import OptionParser

# True by default when running as a script
# Otherwise, we turn the noise off...
verbose = True

def barf(msg):
   print >> sys.stderr, "Error! %s" % msg
   sys.exit(1)

if sys.version_info < (2, 4):
   barf("Requires Python 2.4+")

def parseuri(uri):
   """Parse URI, return (host, port, path) tuple.

   >>> parseuri('http://example.org/testing?somequery#frag')
   ('example.org', 80, '/testing?somequery')
   >>> parseuri('http://example.net:8080/test.html')
   ('example.net', 8080, '/test.html')
   """

   scheme, netplace, path, query, fragid = urlparse.urlsplit(uri)

   if ':' in netplace:
      host, port = netplace.split(':', 2)
      port = int(port)
   else: host, port = netplace, 80

   if query: path += '?' + query

   return host, port, path

def putfile(f, uri, username=None, password=None):
   """HTTP PUT the file f to uri, with optional auth data."""
   host, port, path = parseuri(uri)

   redirect = set([301, 302, 307])
   authenticate = set([401])
   okay = set([200, 201, 204])

   authorized = False
   authorization = None
   tries = 0

   while True:
      # Attempt to HTTP PUT the data
      h = httplib.HTTPConnection(host, port)

      h.putrequest('PUT', path)

      h.putheader('User-Agent', 'put.py/1.0')
      h.putheader('Connection', 'keep-alive')
      h.putheader('Transfer-Encoding', 'chunked')
      h.putheader('Expect', '100-continue')
      h.putheader('Accept', '*/*')
      if authorization:
         h.putheader('Authorization', authorization)
      h.endheaders()

      # Chunked transfer encoding
      # Cf. 'All HTTP/1.1 applications MUST be able to receive and
      # decode the "chunked" transfer-coding'
      # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
      while True:
         bytes = f.read(2048)
         if not bytes: break
         length = len(bytes)
         h.send('%X\r\n' % length)
         h.send(bytes + '\r\n')
      h.send('0\r\n\r\n')
      f.seek(0);

      resp = h.getresponse()
      status = resp.status # an int

      # Got a response, now decide how to act upon it
      if status in redirect:
         location = resp.getheader('Location')
         uri = urlparse.urljoin(uri, location)
         host, port, path = parseuri(uri)

         # We may have to authenticate again
         if authorization:
            authorization = None

      elif status in authenticate:
         # If we've done this already, break
         if authorization:
            # barf("Going around in authentication circles")

           barf("Authentication failed")

         if not (username and password):
           barf("Need a username and password to authenticate with")

         # Get the scheme: Basic or Digest?
         wwwauth = resp.msg['www-authenticate'] # We may need this again
         wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
         wauth = wwwauth.replace('\t', ' ')
         i = wauth.index(' ')
         scheme = wauth[:i].lower()

         if scheme in set(['basic', 'digest','Basic','Digest']):
            if verbose:
               msg = "Performing %s Authentication..." % scheme.capitalize()
         else: barf("Unknown authentication scheme: %s" % scheme)

         if scheme == 'basic' or scheme == 'Basic':
            userpass = username + ':' + password
            userpass = userpass.encode('base64').strip()
            authorized, authorization = True, 'Basic ' + userpass

         elif scheme == 'digest':
            if verbose:
               msg = "uses fragile, undocumented features in urllib2"

               print >> sys.stderr, "Warning! Digest Auth %s" % msg

            import urllib2 # See warning above

            passwd = type('Password', (object,), {
               'find_user_password': lambda self, *args: (username, password),
               'add_password': lambda self, *args: None
            })()

            req = type('Request', (object,), {
               'get_full_url': lambda self: uri,
               'has_data': lambda self: None,
               'get_method': lambda self: 'PUT',
               'get_selector': lambda self: path
            })()

            # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
            auth = urllib2.AbstractDigestAuthHandler(passwd)
            token, challenge = wwwauth.split(' ', 1)
            chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
            userpass = auth.get_authorization(req, chal)
            authorized, authorization = True, 'Digest ' + userpass

      elif status in okay:
         if (username and password) and (not authorized):
            msg = "Warning! The supplied username and password went unused"
            print msg

         if verbose:
            resultLine = "Success! Resource %s"
            statuses = {200: 'modified', 201: 'created', 204: 'modified'}
            print resultLine % statuses[status]

            statusLine = "Response-Status: %s %s"
            print statusLine % (status, resp.reason)

            body = resp.read(58)
            body = body.rstrip('\r\n')
            body = body.encode('string_escape')

            if len(body) >= 58:
               body = body[:57] + '[...]'

            bodyLine = 'Response-Body: "%s"'
            print bodyLine % body
         break

      # @@ raise PutError, do the catching in main?
      else: barf('Got "%s %s"' % (status, resp.reason))

      tries += 1
      if tries >= 50:
         barf("Too many redirects")

   return status, resp

def putname(fn, uri, username=None, password=None):
   """HTTP PUT the file with filename fn to uri, with optional auth data."""
   auth = {'username': username, 'password': password}

   if fn != '-':
      f = open(fn, 'rb')
      status, resp = putfile(f, uri, **auth)
      f.close()
   else: status, resp = putfile(sys.stdin, uri, **auth)

   return status, resp

def put(s, uri, username=None, password=None):
   """HTTP PUT the string s to uri, with optional auth data."""
   try: from cStringIO import StringIO
   except ImportError:
      from StringIO import StringIO

   f = StringIO(s)
   f.seek(0)
   status, resp = putfile(f, uri, username=username, password=password)
   f.close()

   return status, conn

def main(argv=None):
   usage = ('%prog [options] filename uri\n' +
            'The filename may be - for stdin\n' +
            'Use command line password at your own risk!')

   parser = OptionParser(usage=usage)
   parser.add_option('-u', '--username', help='HTTP Auth username')
   parser.add_option('-p', '--password', help='HTTP Auth password')
   parser.add_option('-q', '--quiet', action='store_true', help="shhh!")
   options, args = parser.parse_args(argv)

   if len(args) != 2:
      parser.error("Requires two arguments, filename and uri")

   fn, uri = args
   if not uri.startswith('http:'):
      parser.error("The uri argument must start with 'http:'")

   if options.username and not options.password or \
      options.password and not options.username:
      parser.error("Must have both username and password or neither")

   global verbose
   verbose = (not options.quiet)

   auth = {'username': options.username, 'password': options.password}
   putname(fn, uri, **auth)

if __name__ == '__main__':
   main()