CDash:Dart2AndCDashSubmission
Setting up dual Dart2 and CDash Submissions
If you are moving from Dart2 to CDash you may want to submit to both dashboards in parallel for a while, before switching solely to CDash.
The following shows how to setup CTest to use a trigger script, which in turn does a submit to both Dart2 (via XML-RPC) and CDash (via HTTP).
Note: This method doesn't support submission of Dart2 notes files.
Configure CTest to use a trigger script:
SET (CTEST_DROP_METHOD "http") SET (CTEST_DROP_SITE "mysite.org") SET (CTEST_DROP_LOCATION "/cgi-bin/HTTPUploadDartFile.cgi") SET (CTEST_TRIGGER_SITE "http://${DROP_SITE}/cgi-bin/TriggerSiteDart2AndCDash.cgi")
Both HTTPUploadDartFile.cgi and TriggerSiteDart2AndCDash.cgi have a hard coded path that may require editing: /srv/dart2/incoming
This current implementation only supports a single hard coded project in TriggerSiteDart2AndCDash.cgi. You can of course create a specific trigger site per project to workaround this.
HTTPUploadDartFile.cgi
#!/usr/bin/env python # # Script that will upload files to specified directory on the server, where # triggering script will find it. # # Installation: # Place this script in your cgi-bin area. # Change the variable "incoming_directory" to match your # installation. ### import cgi import re import sys import os import string print "Content-type: text/html" print done = 0 incoming_directory = '/srv/dart2/incoming' form = cgi.FieldStorage() # Debug #try: # tmpfile = open("/tmp/http_upload_log_file.log", "w") # tmpfile.write(`form`) # tmpfile.close() #except Exception: pass FileData = "" FileName = "" # process form if type(form.value) == type("foo"): if "QUERY_STRING" in os.environ: dt = cgi.parse_qs(os.environ["QUERY_STRING"]) if "FileName" in dt: FileName = dt["FileName"][0] FileData = form.value elif "FileName" in form: FileName = form["FileName"].value if "FileData" in form: FileData = "form" # verify file specified if not FileData or not FileName: print "Please send file. " print "FileName has to contain file name" print "FileData has to contain file content" print form sys.exit(0) print "Received file: " + FileName # check if valid file name filename = re.sub("[/\\|:%%]", "_", FileName) if re.match(".+___.+___[0-9]{8}-[0-9]{4}-(Experimental|Nightly|Continuous)___XML___(Update|Configure|Build|Test|Coverage|Purify).xml", filename): print "Correct file name" else: print "Can only upload files with format:" print "<site>___<BuildType>___<TAG>-<TestType>___XML___<File>.xml" sys.exit(0) # get the file data file_lines = "" if FileData == "form": fileitem = form["FileData"] if fileitem.file: file_lines = fileitem.file.read() else: print "Not a file" else: file_lines = FileData # write to a file on disk if file_lines: file = open(incoming_directory + "/" + filename, "w") if not file: print "Cannot create file: %s/%s" % ( incoming_directory, filename ) sys.exit(0) file.write(file_lines) file.close() print "Thank you for the file" done = 1 if not done: print "Problem summiting data"
TriggerSiteDart2AndCDash.cgi
Note: If the script fails it will create log files in /tmp/tmp*.html.
#!/usr/bin/env python # # Script that will resubmit incoming testing results to # Dart2 (via XMLRPC) and CDash (via HTTP). # # This scripts intended use is to allow the transition from # Dart2 to CDash, by allowing both to be run in parallel. # ### import cgi import cgitb; cgitb.enable(display=0, logdir="/tmp") import sys import os import xmlrpclib import put def Trigger(dart2Url, cdashUrl, projectName, dropLocation): print "Content-Type: text/html" print form = cgi.FieldStorage() xmlfile = "" if "xmlfile" in form: xmlfile = form["xmlfile"].value if not xmlfile: print "No submission file" sys.exit(1) ipfile = dropLocation + xmlfile try: dart2Server = xmlrpclib.ServerProxy(dart2Url + projectName + "/Command/") try: fp = open(ipfile) content = fp.read() bin = xmlrpclib.Binary(content) print "Server responded: [%s]" % dart2Server.Submit.put(bin) fp.close() except Exception, v: print "ERROR", v print "Unexpected error:", sys.exc_info() except: print "Problem submitting XML-RPC for the file: %s" % xmlfile try: cdashServer = cdashUrl + "CDash/submit.php?project=" + projectName f = open(ipfile, 'rb') put.putfile(f, cdashServer) f.close() except: print "Problem submitting HTTP for the file: %s" % xmlfile os.unlink(ipfile) if __name__ == "__main__": # Ensure all paths have a trailing slash # Dart2 URL, CDash URL, project, incoming directory Trigger("http://localhost:8081/", "http://localhost/", "MyProject", "/srv/dart2/incoming/")
put.py
Note: This version of put requires Apache2 (for chunking).
#!/usr/bin/env python """ put.py - Python HTTP PUT Client Author: Sean B. Palmer, inamidst.com Basic API usage, once with optional auth: import put put.putname('test.txt', 'http://example.org/test') f = open('test.txt', 'rb') put.putfile(f, 'http://example.org/test') f.close() bytes = open('test.txt', 'rb').read() auth = {'username': 'myuser', 'password': 'mypass'} put.put(bytes, 'http://example.org/test', **auth) """ import sys, httplib, urlparse from optparse import OptionParser # True by default when running as a script # Otherwise, we turn the noise off... verbose = True def barf(msg): print >> sys.stderr, "Error! %s" % msg sys.exit(1) if sys.version_info < (2, 4): barf("Requires Python 2.4+") def parseuri(uri): """Parse URI, return (host, port, path) tuple. >>> parseuri('http://example.org/testing?somequery#frag') ('example.org', 80, '/testing?somequery') >>> parseuri('http://example.net:8080/test.html') ('example.net', 8080, '/test.html') """ scheme, netplace, path, query, fragid = urlparse.urlsplit(uri) if ':' in netplace: host, port = netplace.split(':', 2) port = int(port) else: host, port = netplace, 80 if query: path += '?' + query return host, port, path def putfile(f, uri, username=None, password=None): """HTTP PUT the file f to uri, with optional auth data.""" host, port, path = parseuri(uri) redirect = set([301, 302, 307]) authenticate = set([401]) okay = set([200, 201, 204]) authorized = False authorization = None tries = 0 while True: # Attempt to HTTP PUT the data h = httplib.HTTPConnection(host, port) h.putrequest('PUT', path) h.putheader('User-Agent', 'put.py/1.0') h.putheader('Connection', 'keep-alive') h.putheader('Transfer-Encoding', 'chunked') h.putheader('Expect', '100-continue') h.putheader('Accept', '*/*') if authorization: h.putheader('Authorization', authorization) h.endheaders() # Chunked transfer encoding # Cf. 'All HTTP/1.1 applications MUST be able to receive and # decode the "chunked" transfer-coding' # - http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html while True: bytes = f.read(2048) if not bytes: break length = len(bytes) h.send('%X\r\n' % length) h.send(bytes + '\r\n') h.send('0\r\n\r\n') f.seek(0); resp = h.getresponse() status = resp.status # an int # Got a response, now decide how to act upon it if status in redirect: location = resp.getheader('Location') uri = urlparse.urljoin(uri, location) host, port, path = parseuri(uri) # We may have to authenticate again if authorization: authorization = None elif status in authenticate: # If we've done this already, break if authorization: # barf("Going around in authentication circles") barf("Authentication failed") if not (username and password): barf("Need a username and password to authenticate with") # Get the scheme: Basic or Digest? wwwauth = resp.msg['www-authenticate'] # We may need this again wauth = wwwauth.lstrip(' \t') # Hence use wauth not wwwauth here wauth = wwwauth.replace('\t', ' ') i = wauth.index(' ') scheme = wauth[:i].lower() if scheme in set(['basic', 'digest','Basic','Digest']): if verbose: msg = "Performing %s Authentication..." % scheme.capitalize() else: barf("Unknown authentication scheme: %s" % scheme) if scheme == 'basic' or scheme == 'Basic': import base64 userpass = username + ':' + password userpass = base64.encodestring(userpass).strip() authorized, authorization = True, 'Basic ' + userpass elif scheme == 'digest': if verbose: msg = "uses fragile, undocumented features in urllib2" print >> sys.stderr, "Warning! Digest Auth %s" % msg import urllib2 # See warning above passwd = type('Password', (object,), { 'find_user_password': lambda self, *args: (username, password), 'add_password': lambda self, *args: None })() req = type('Request', (object,), { 'get_full_url': lambda self: uri, 'has_data': lambda self: None, 'get_method': lambda self: 'PUT', 'get_selector': lambda self: path })() # Cf. urllib2.AbstractDigestAuthHandler.retry_http_digest_auth auth = urllib2.AbstractDigestAuthHandler(passwd) token, challenge = wwwauth.split(' ', 1) chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) userpass = auth.get_authorization(req, chal) authorized, authorization = True, 'Digest ' + userpass elif status in okay: if (username and password) and (not authorized): msg = "Warning! The supplied username and password went unused" print msg if verbose: resultLine = "Success! Resource %s" statuses = {200: 'modified', 201: 'created', 204: 'modified'} print resultLine % statuses[status] statusLine = "Response-Status: %s %s" print statusLine % (status, resp.reason) body = resp.read(58) body = body.rstrip('\r\n') body = body.encode('string_escape') if len(body) >= 58: body = body[:57] + '[...]' bodyLine = 'Response-Body: "%s"' print bodyLine % body break # @@ raise PutError, do the catching in main? else: barf('Got "%s %s"' % (status, resp.reason)) tries += 1 if tries >= 50: barf("Too many redirects") return status, resp def putname(fn, uri, username=None, password=None): """HTTP PUT the file with filename fn to uri, with optional auth data.""" auth = {'username': username, 'password': password} if fn != '-': f = open(fn, 'rb') status, resp = putfile(f, uri, **auth) f.close() else: status, resp = putfile(sys.stdin, uri, **auth) return status, resp def put(s, uri, username=None, password=None): """HTTP PUT the string s to uri, with optional auth data.""" try: from cStringIO import StringIO except ImportError: from StringIO import StringIO f = StringIO(s) f.seek(0) status, resp = putfile(f, uri, username=username, password=password) f.close() return status, conn def main(argv=None): usage = ('%prog [options] filename uri\n' + 'The filename may be - for stdin\n' + 'Use command line password at your own risk!') parser = OptionParser(usage=usage) parser.add_option('-u', '--username', help='HTTP Auth username') parser.add_option('-p', '--password', help='HTTP Auth password') parser.add_option('-q', '--quiet', action='store_true', help="shhh!") options, args = parser.parse_args(argv) if len(args) != 2: parser.error("Requires two arguments, filename and uri") fn, uri = args if not uri.startswith('http:'): parser.error("The uri argument must start with 'http:'") if ((options.username and not options.password) or (options.password and not options.username)): parser.error("Must have both username and password or neither") global verbose verbose = (not options.quiet) auth = {'username': options.username, 'password': options.password} putname(fn, uri, **auth) if __name__ == '__main__': main()