#!/usr/bin/env python2 import sys, os, zipfile # One package must be submitted for each run, for up to 3 runs. # A submission package is a zip file containing two files: # - cccs--.tsv containing the output of the system for that run # - cccs--.README containing a description of the run # The format for the system output file is one synopsis per line with the following tab-separated format: # \t : \t \n if len(sys.argv) != 4: print 'USAGE: %s ' % sys.argv[0] sys.exit(1) team_id = sys.argv[1] run_id = sys.argv[2] submission_package = sys.argv[3] run_filename = 'cccs-%s-%s.tsv' % (team_id, run_id) description_filename = 'cccs-%s-%s.README' % (team_id, run_id) def die(message): print message sys.exit(1) if not os.path.exists(submission_package): die('ERROR: could not find submission package "%s"' % submission_package) try: package_fp = zipfile.ZipFile(submission_package, 'r') files = package_fp.namelist() #if len(files) != 2: # die('ERROR: zip package should contain exactly two files') if run_filename not in files: die('ERROR: zip package should contain a file named %s' % run_filename) if description_filename not in files: die('ERROR: zip package should contain a file named %s' % description_filename) description_fp = package_fp.open(description_filename) if len(description_fp.readlines()) < 1: die('ERROR: description file should contain some text') description_fp.close() run_fp = package_fp.open(run_filename) seen = {} for line_num, line in enumerate(run_fp): tokens = line.strip().split('\t') if len(tokens) != 3: die('ERROR: line %d does not contain exactly 3 tab-separated fields' % (line_num + 1)) if len(tokens[0]) == 0: die('ERROR: empty conversation id at line %d' % (line_num + 1)) if len(tokens[2]) == 0: print 'WARNING: empty synopsis at line %d' % (line_num + 1) if tokens[1] != '%s:%s' % (team_id, run_id): die('ERROR: invalid system identifier "%s" at line %d, should be "%s:%s"' % (tokens[1], line_num + 1, team_id, run_id)) if tokens[0] in seen: die('ERROR: more than one synopsis for conversation id "%s", at line %d, previous was at line %d' % (tokens[0], line_num + 1, seen[tokens[0]])) else: seen[tokens[0]] = line_num + 1 print 'INFO: %d synopses found in this submission' % len(seen) run_fp.close() package_fp.close() except Exception as e: die('ERROR: exception while processing submission package "%s"' % str(e)) print 'INFO: all checks successful'