135 lines
4.4 KiB
Python
135 lines
4.4 KiB
Python
from __future__ import print_function
|
|
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import re
|
|
|
|
import scriptCommon
|
|
from scriptCommon import catchPath
|
|
|
|
rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
|
|
|
|
filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
|
|
filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
|
|
pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
|
|
lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
|
|
hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
|
|
durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
|
|
versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
|
|
devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
|
|
nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
|
|
exeNameParser = re.compile(r'(.*?)\b(CatchSelfTest|SelfTest)\b(.*)')
|
|
|
|
if len(sys.argv) == 2:
|
|
cmdPath = sys.argv[1]
|
|
else:
|
|
cmdPath = scriptCommon.getBuildExecutable()
|
|
|
|
if not cmdPath.startswith("/"):
|
|
cmdPath = os.path.join(catchPath, cmdPath)
|
|
|
|
overallResult = 0
|
|
|
|
def filterLine(line):
|
|
m = filenameParser.match(line)
|
|
if m:
|
|
line = m.group(2) + m.group(3)
|
|
m2 = filelineParser.match(line)
|
|
if m2:
|
|
line = m2.group(1) + "<line number>" + m2.group(3)
|
|
else:
|
|
m2 = lineNumberParser.match(line)
|
|
if m2:
|
|
line = m2.group(1) + m2.group(2)
|
|
m = pathParser.match(line)
|
|
if m:
|
|
path = "/" + m.group(2)
|
|
if path.startswith(catchPath):
|
|
path = path[1 + len(catchPath):]
|
|
line = m.group(1) + path + m.group(3)
|
|
m = devVersionParser.match(line)
|
|
if m:
|
|
line = m.group(1) + "<version>" + m.group(2)
|
|
else:
|
|
m = versionParser.match(line)
|
|
if m:
|
|
line = m.group(1) + "<version>" + m.group(2)
|
|
|
|
m = nullParser.match(line)
|
|
if m:
|
|
line = m.group(1) + "0" + m.group(3)
|
|
|
|
m = exeNameParser.match(line)
|
|
if m:
|
|
line = m.group(1) + "<exe-name>" + m.group(3)
|
|
|
|
while True:
|
|
m = hexParser.match(line)
|
|
if m:
|
|
line = m.group(1) + "0x<hex digits>" + m.group(3)
|
|
else:
|
|
break
|
|
m = durationsParser.match(line)
|
|
if m:
|
|
line = m.group(1) + 'time="{duration}"' + m.group(2)
|
|
return line
|
|
|
|
|
|
def approve(baseName, args):
|
|
global overallResult
|
|
args[0:0] = [cmdPath]
|
|
if not os.path.exists(cmdPath):
|
|
raise Exception("Executable doesn't exist at " + cmdPath)
|
|
baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
|
|
rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
|
|
filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
|
|
|
|
f = open(rawResultsPath, 'w')
|
|
subprocess.call(args, stdout=f, stderr=f)
|
|
f.close()
|
|
|
|
rawFile = open(rawResultsPath, 'r')
|
|
filteredFile = open(filteredResultsPath, 'w')
|
|
for line in rawFile:
|
|
filteredFile.write(filterLine(line).rstrip() + "\n")
|
|
filteredFile.close()
|
|
rawFile.close()
|
|
|
|
os.remove(rawResultsPath)
|
|
print()
|
|
print(baseName + ":")
|
|
if os.path.exists(baselinesPath):
|
|
diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
|
|
if diffResult == 0:
|
|
os.remove(filteredResultsPath)
|
|
print(" \033[92mResults matched")
|
|
else:
|
|
print(" \n****************************\n \033[91mResults differed")
|
|
if diffResult > overallResult:
|
|
overallResult = diffResult
|
|
print("\033[0m")
|
|
else:
|
|
print(" first approval")
|
|
if overallResult == 0:
|
|
overallResult = 1
|
|
|
|
|
|
print("Running approvals against executable:")
|
|
print(" " + cmdPath)
|
|
|
|
# Standard console reporter
|
|
approve("console.std", ["~[c++11]", "--order", "lex"])
|
|
# console reporter, include passes, warn about No Assertions
|
|
approve("console.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "--order", "lex"])
|
|
# console reporter, include passes, warn about No Assertions, limit failures to first 4
|
|
approve("console.swa4", ["~[c++11]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
|
|
# junit reporter, include passes, warn about No Assertions
|
|
approve("junit.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
|
|
# xml reporter, include passes, warn about No Assertions
|
|
approve("xml.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
|
|
|
|
if overallResult != 0:
|
|
print("If these differenecs are expected run approve.py to approve new baselines")
|
|
exit(overallResult)
|