CDash:Dart2AndCDashSubmission
From KitwarePublic
Contents |
Setting up dual Dart2 and CDash Submissions
If you are moving from Dart2 to CDash you may want to submit to both dashboards in parallel for a while, before switching solely to CDash.
The following shows how to setup CTest to use a trigger script, which in turn does a submit to both Dart2 (via XML-RPC) and CDash (via HTTP).
Note: This method doesn't support submission of Dart2 notes files.
Configure CTest to use a trigger script:
SET (CTEST_DROP_METHOD "http") SET (CTEST_DROP_SITE "mysite.org") SET (CTEST_DROP_LOCATION "/cgi-bin/HTTPUploadDartFile.cgi") SET (CTEST_TRIGGER_SITE "http://${DROP_SITE}/cgi-bin/TriggerSiteDart2AndCDash.cgi")
Both HTTPUploadDartFile.cgi and TriggerSiteDart2AndCDash.cgi have a hard coded path that may require editing: /srv/dart2/incoming
This current implementation only supports a single hard coded project in TriggerSiteDart2AndCDash.cgi. You can of course create a specific trigger site per project to workaround this.
HTTPUploadDartFile.cgi
#!/usr/bin/env python
#
# Script that will upload files to specified directory on the server, where
# triggering script will find it.
#
# Installation:
# Place this script in your cgi-bin area.
# Change the variable "incoming_directory" to match your
# installation.
###
import cgi
import re
import sys
import os
import string
print "Content-type: text/html"
print
done = 0
incoming_directory = '/srv/dart2/incoming'
form = cgi.FieldStorage()
# Debug
#try:
# tmpfile = open("/tmp/http_upload_log_file.log", "w")
# tmpfile.write(`form`)
# tmpfile.close()
#except Exception: pass
FileData = ""
FileName = ""
# process form
if type(form.value) == type("foo"):
if os.environ.has_key("QUERY_STRING"):
dt = cgi.parse_qs(os.environ["QUERY_STRING"])
if dt.has_key("FileName"):
FileName = dt["FileName"][0]
FileData = form.value
elif form.has_key("FileName"):
FileName = form["FileName"].value
if form.has_key("FileData"):
FileData = "form"
# verify file specified
if not FileData or not FileName:
print "Please send file. "
print "FileName has to contain file name"
print "FileData has to contain file content"
print form
sys.exit(0)
print "Received file: " + FileName
# check if valid file name
filename = re.sub("[/\\|:%%]", "_", FileName)
if re.match(".+___.+___[0-9]{8}-[0-9]{4}-(Experimental|Nightly|Continuous)___XML___(Update|Configure|Build|Test|Coverage|Purify).xml", filename):
print "Correct file name"
else:
print "Can only upload files with format:"
print "<site>___<BuildType>___<TAG>-<TestType>___XML___<File>.xml"
sys.exit(0)
# get the file data
file_lines = ""
if FileData == "form":
fileitem = form["FileData"]
if fileitem.file:
file_lines = fileitem.file.read()
else:
print "Not a file"
else:
file_lines = FileData
# write to a file on disk
if file_lines:
file = open(incoming_directory + "/" + filename, "w")
if not file:
print "Cannot create file: %s/%s" % ( incoming_directory, filename )
sys.exit(0)
file.write(file_lines)
file.close()
print "Thank you for the file"
done = 1
if not done:
print "Problem summiting data"
TriggerSiteDart2AndCDash.cgi
Note: If the script fails it will create log files in /tmp/tmp*.html.
#!/usr/bin/env python
#
# Script that will resubmit incoming testing results to
# Dart2 (via XMLRPC) and CDash (via HTTP).
#
# This scripts intended use is to allow the transition from
# Dart2 to CDash, by allowing both to be run in parallel.
#
###
import cgi
import cgitb; cgitb.enable(display=0, logdir="/tmp")
import sys
import os
import xmlrpclib
import put
def Trigger(dart2Url, cdashUrl, projectName, dropLocation):
print "Content-Type: text/html"
print
form = cgi.FieldStorage()
xmlfile = ""
if form.has_key("xmlfile"):
xmlfile = form["xmlfile"].value
if not xmlfile:
print "No submission file"
sys.exit(1)
ipfile = dropLocation + xmlfile
try:
dart2Server = xmlrpclib.ServerProxy(dart2Url + projectName + "/Command/")
try:
fp = open(ipfile)
content = fp.read()
bin = xmlrpclib.Binary(content)
print "Server responded: [%s]" % dart2Server.Submit.put(bin)
fp.close()
except Exception, v:
print "ERROR", v
print "Unexpected error:", sys.exc_info()
except:
print "Problem submitting XML-RPC for the file: %s" % xmlfile
try:
cdashServer = cdashUrl + "CDash/submit.php?project=" + projectName
f = open(ipfile, 'rb')
put.putfile(f, cdashServer)
f.close()
except:
print "Problem submitting HTTP for the file: %s" % xmlfile
os.unlink(ipfile)
if __name__ == "__main__":
# Ensure all paths have a trailing slash
# Dart2 URL, CDash URL, project, incoming directory
Trigger("http://localhost:8081/", "http://localhost/", "MyProject", "/srv/dart2/incoming/")
put.py
Note: This version of put requires Apache2 (for chunking).
#!/usr/bin/env python
"""
put.py - Python HTTP PUT Client
Author: Sean B. Palmer, inamidst.com
Basic API usage, once with optional auth:
import put
put.putname('test.txt', 'http://example.org/test')
f = open('test.txt', 'rb')
put.putfile(f, 'http://example.org/test')
f.close()
bytes = open('test.txt', 'rb').read()
auth = {'username': 'myuser', 'password': 'mypass'}
put.put(bytes, 'http://example.org/test', **auth)
"""
import sys, httplib, urlparse
from optparse import OptionParser
# True by default when running as a script
# Otherwise, we turn the noise off...
verbose = True
def barf(msg):
print >> sys.stderr, "Error! %s" % msg
sys.exit(1)
if sys.version_info < (2, 4):
barf("Requires Python 2.4+")
def parseuri(uri):
"""Parse URI, return (host, port, path) tuple.
>>> parseuri('http://example.org/testing?somequery#frag')
('example.org', 80, '/testing?somequery')
>>> parseuri('http://example.net:8080/test.html')
('example.net', 8080, '/test.html')
"""
scheme, netplace, path, query, fragid = urlparse.urlsplit(uri)
if ':' in netplace:
host, port = netplace.split(':', 2)
port = int(port)
else: host, port = netplace, 80
if query: path += '?' + query
return host, port, path
def putfile(f, uri, username=None, password=None):
"""HTTP PUT the file f to uri, with optional auth data."""
host, port, path = parseuri(uri)
redirect = set([301, 302, 307])
authenticate = set([401])
okay = set([200, 201, 204])
authorized = False
authorization = None
tries = 0
while True:
# Attempt to HTTP PUT the data
h = httplib.HTTPConnection(host, port)
h.putrequest('PUT', path)
h.putheader('User-Agent', 'put.py/1.0')
h.putheader('Connection', 'keep-alive')
h.putheader('Transfer-Encoding', 'chunked')
h.putheader('Expect', '100-continue')
h.putheader('Accept', '*/*')
if authorization:
h.putheader('Authorization', authorization)
h.endheaders()
# Chunked transfer encoding
# Cf. 'All HTTP/1.1 applications MUST be able to receive and
# decode the "chunked" transfer-coding'
# - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html
while True:
bytes = f.read(2048)
if not bytes: break
length = len(bytes)
h.send('%X\r\n' % length)
h.send(bytes + '\r\n')
h.send('0\r\n\r\n')
f.seek(0);
resp = h.getresponse()
status = resp.status # an int
# Got a response, now decide how to act upon it
if status in redirect:
location = resp.getheader('Location')
uri = urlparse.urljoin(uri, location)
host, port, path = parseuri(uri)
# We may have to authenticate again
if authorization:
authorization = None
elif status in authenticate:
# If we've done this already, break
if authorization:
# barf("Going around in authentication circles")
barf("Authentication failed")
if not (username and password):
barf("Need a username and password to authenticate with")
# Get the scheme: Basic or Digest?
wwwauth = resp.msg['www-authenticate'] # We may need this again
wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here
wauth = wwwauth.replace('\t', ' ')
i = wauth.index(' ')
scheme = wauth[:i].lower()
if scheme in set(['basic', 'digest','Basic','Digest']):
if verbose:
msg = "Performing %s Authentication..." % scheme.capitalize()
else: barf("Unknown authentication scheme: %s" % scheme)
if scheme == 'basic' or scheme == 'Basic':
import base64
userpass = username + ':' + password
userpass = base64.encodestring(userpass).strip()
authorized, authorization = True, 'Basic ' + userpass
elif scheme == 'digest':
if verbose:
msg = "uses fragile, undocumented features in urllib2"
print >> sys.stderr, "Warning! Digest Auth %s" % msg
import urllib2 # See warning above
passwd = type('Password', (object,), {
'find_user_password': lambda self, *args: (username, password),
'add_password': lambda self, *args: None
})()
req = type('Request', (object,), {
'get_full_url': lambda self: uri,
'has_data': lambda self: None,
'get_method': lambda self: 'PUT',
'get_selector': lambda self: path
})()
# Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth
auth = urllib2.AbstractDigestAuthHandler(passwd)
token, challenge = wwwauth.split(' ', 1)
chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge))
userpass = auth.get_authorization(req, chal)
authorized, authorization = True, 'Digest ' + userpass
elif status in okay:
if (username and password) and (not authorized):
msg = "Warning! The supplied username and password went unused"
print msg
if verbose:
resultLine = "Success! Resource %s"
statuses = {200: 'modified', 201: 'created', 204: 'modified'}
print resultLine % statuses[status]
statusLine = "Response-Status: %s %s"
print statusLine % (status, resp.reason)
body = resp.read(58)
body = body.rstrip('\r\n')
body = body.encode('string_escape')
if len(body) >= 58:
body = body[:57] + '[...]'
bodyLine = 'Response-Body: "%s"'
print bodyLine % body
break
# @@ raise PutError, do the catching in main?
else: barf('Got "%s %s"' % (status, resp.reason))
tries += 1
if tries >= 50:
barf("Too many redirects")
return status, resp
def putname(fn, uri, username=None, password=None):
"""HTTP PUT the file with filename fn to uri, with optional auth data."""
auth = {'username': username, 'password': password}
if fn != '-':
f = open(fn, 'rb')
status, resp = putfile(f, uri, **auth)
f.close()
else: status, resp = putfile(sys.stdin, uri, **auth)
return status, resp
def put(s, uri, username=None, password=None):
"""HTTP PUT the string s to uri, with optional auth data."""
try: from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
f = StringIO(s)
f.seek(0)
status, resp = putfile(f, uri, username=username, password=password)
f.close()
return status, conn
def main(argv=None):
usage = ('%prog [options] filename uri\n' +
'The filename may be - for stdin\n' +
'Use command line password at your own risk!')
parser = OptionParser(usage=usage)
parser.add_option('-u', '--username', help='HTTP Auth username')
parser.add_option('-p', '--password', help='HTTP Auth password')
parser.add_option('-q', '--quiet', action='store_true', help="shhh!")
options, args = parser.parse_args(argv)
if len(args) != 2:
parser.error("Requires two arguments, filename and uri")
fn, uri = args
if not uri.startswith('http:'):
parser.error("The uri argument must start with 'http:'")
if ((options.username and not options.password) or
(options.password and not options.username)):
parser.error("Must have both username and password or neither")
global verbose
verbose = (not options.quiet)
auth = {'username': options.username, 'password': options.password}
putname(fn, uri, **auth)
if __name__ == '__main__':
main()
