summaryrefslogtreecommitdiff
path: root/runtests.py
blob: e7ab87b88607eb11ad0207901364bb10fbede20f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import glob
import os
import subprocess

SPIM_COPYRIGHT_HEADER = '''SPIM Version 7.3. of August 28, 2006
Copyright 1990-2004 by James R. Larus (larus@cs.wisc.edu).
All Rights Reserved.
See the file README for a full copyright notice.
Loaded: /usr/lib/spim/exceptions.s
'''

def bold(s):
    return "\033[1;37m%s\033[0m" % s

OK = "\033[1;32mOK\033[0m"
NOTOK = "\033[1;31m!!\033[0m"

def list_tests():
    tests = []
    for path in ("lexing", "parsing", "typing", "assembly"):
        tests += glob.glob("tests/%s/*.ml" % path)
    return tests

def run_test(test_file):
    f = open(test_file)
    lines = f.readlines()
    f.close()
    expected_return_code = int(lines[1][13])
    if expected_return_code != 0:
        expected_error = ""
        i = 3
        while not lines[i].startswith("*)"):
            expected_error += lines[i]
            i += 1
    base_command = "./petit-caml %s" % test_file
    test_output = False
    test_assembly = False
    if "lexing" in test_file or "parsing" in test_file:
        command = base_command + " -parse-only"
    elif "typing" in test_file:
        test_output = True
        command = base_command + " -type-only -print-decl-types"
    else:
        test_assembly = True
        command = base_command
    process = subprocess.Popen(command,
                               stderr = subprocess.PIPE,
                               stdout = subprocess.PIPE,
                               shell = True)
    return_code = process.wait()
    if return_code != 0:
        error = process.stderr.read()
    status = OK if return_code == expected_return_code else NOTOK
    if return_code != 0:
        if status == OK:
            if expected_error == error:
                status = OK
            else:
                status = "%s\n%s\n%s\n%s" % (NOTOK, error.strip(),
                                             bold("Expected :"),
                                             expected_error.strip())
        else:
            status = "%s\n%s" % (NOTOK, error.strip())
    if return_code == 0 and (test_output or test_assembly):
        i = 3
        expected_output = ""
        while not lines[i].startswith("*)"):
            expected_output += lines[i]
            i += 1
    if status == OK and return_code == 0 and test_output:
        output = process.stdout.read()
        if output != expected_output:
            status = "%s\n%s\n%s\n%s" % (NOTOK, output.strip(),
                                         bold("Expected :"),
                                         expected_output.strip())
    if status == OK and return_code == 0 and test_assembly:
        process = subprocess.Popen("spim %s" % test_file.replace(".ml", ".s"),
                                   stderr = subprocess.PIPE,
                                   stdout = subprocess.PIPE,
                                   shell = True)
        return_code = process.wait()
        if return_code != 0:
            error = process.stderr.read()
            status = "%s\n%s" % (NOTOK, error.strip())
        output = process.stdout.read()
        output = output.replace(SPIM_COPYRIGHT_HEADER, "")
        if output != expected_output:
            status = "%s\n%s\n%s\n%s" % (NOTOK, output.strip(),
                                         bold("Expected :"),
                                         expected_output.strip())
    print "%s : %s" % (bold(test_file), status)

def run_tests():
    tests = list_tests()
    tests.sort()
    map(run_test, tests)

if __name__ == "__main__":
    if not os.path.isfile("./petit-caml"):
        print NOTOK, \
              bold("Petit-Caml unavailable - please build it first"), \
              NOTOK
        raise SystemExit
    run_tests()