[ tor-browser ].git.dasho

process-ogles2-tests.py (16509B)
      1 #! /usr/bin/env python2
      2 
      3 """generates tests from OpenGL ES 2.0 .run/.test files."""
      4 
      5 import os
      6 import os.path
      7 import sys
      8 import re
      9 import json
     10 import shutil
     11 from optparse import OptionParser
     12 from xml.dom.minidom import parse
     13 
     14 if sys.version < '2.6':
     15   print 'Wrong Python Version !!!: Need >= 2.6'
     16   sys.exit(1)
     17 
     18 # each shader test generates up to 3 512x512 images.
     19 # a 512x512 image takes 1meg of memory so set this
     20 # number apporpriate for the platform with
     21 # the smallest memory issue. At 8 that means
     22 # at least 24 meg is needed to run the test.
     23 MAX_TESTS_PER_SET = 8
     24 
     25 VERBOSE = False
     26 
     27 FILTERS = [
     28  re.compile("GL/"),
     29 ]
     30 
     31 LICENSE = """
     32 Copyright (c) 2019 The Khronos Group Inc.
     33 Use of this source code is governed by an MIT-style license that can be
     34 found in the LICENSE.txt file.
     35 """.strip()
     36 
     37 COMMENT_RE = re.compile("/\*\n\*\*\s+Copyright.*?\*/",
     38                        re.IGNORECASE | re.DOTALL)
     39 REMOVE_COPYRIGHT_RE = re.compile("\/\/\s+Copyright.*?\n",
     40                                 re.IGNORECASE | re.DOTALL)
     41 MATRIX_RE = re.compile("Matrix(\\d)")
     42 
     43 VALID_UNIFORM_TYPES = [
     44  "uniform1f",
     45  "uniform1fv",
     46  "uniform1fv",
     47  "uniform1i",
     48  "uniform1iv",
     49  "uniform1iv",
     50  "uniform2f",
     51  "uniform2fv",
     52  "uniform2fv",
     53  "uniform2i",
     54  "uniform2iv",
     55  "uniform2iv",
     56  "uniform3f",
     57  "uniform3fv",
     58  "uniform3fv",
     59  "uniform3i",
     60  "uniform3iv",
     61  "uniform3iv",
     62  "uniform4f",
     63  "uniform4fv",
     64  "uniform4fv",
     65  "uniform4i",
     66  "uniform4iv",
     67  "uniform4ivy",
     68  "uniformMatrix2fv",
     69  "uniformMatrix2fv",
     70  "uniformMatrix3fv",
     71  "uniformMatrix3fv",
     72  "uniformMatrix4fv",
     73  "uniformMatrix4fv",
     74 ]
     75 
     76 SUBSTITUTIONS = [
     77  ("uniformmat3fv", "uniformMatrix3fv"),
     78  ("uniformmat4fv", "uniformMatrix4fv"),
     79 ]
     80 
     81 
     82 def Log(msg):
     83  global VERBOSE
     84  if VERBOSE:
     85    print msg
     86 
     87 
     88 def TransposeMatrix(values, dim):
     89  size = dim * dim
     90  count = len(values) / size
     91  for m in range(0, count):
     92    offset = m * size
     93    for i in range(0, dim):
     94      for j in range(i + 1, dim):
     95        t = values[offset + i * dim + j]
     96        values[offset + i * dim + j] = values[offset + j * dim + i]
     97        values[offset + j * dim + i] = t
     98 
     99 
    100 def GetValidTypeName(type_name):
    101  global VALID_UNIFORM_TYPES
    102  global SUBSTITUTIONS
    103  for subst in SUBSTITUTIONS:
    104    type_name = type_name.replace(subst[0], subst[1])
    105  if not type_name in VALID_UNIFORM_TYPES:
    106    print "unknown type name: ", type_name
    107    raise SyntaxError
    108  return type_name
    109 
    110 
    111 def WriteOpen(filename):
    112  dirname = os.path.dirname(filename)
    113  if len(dirname) > 0 and not os.path.exists(dirname):
    114    os.makedirs(dirname)
    115  return open(filename, "wb")
    116 
    117 
    118 class TxtWriter():
    119  def __init__(self, filename):
    120    self.filename = filename
    121    self.lines = []
    122 
    123  def Write(self, line):
    124    self.lines.append(line)
    125 
    126  def Close(self):
    127    if len(self.lines) > 0:
    128      Log("Writing: %s" % self.filename)
    129      f = WriteOpen(self.filename)
    130      f.write("# this file is auto-generated. DO NOT EDIT.\n")
    131      f.write("".join(self.lines))
    132      f.close()
    133 
    134 
    135 def ReadFileAsLines(filename):
    136  f = open(filename, "r")
    137  lines = f.readlines()
    138  f.close()
    139  return [line.strip() for line in lines]
    140 
    141 
    142 def ReadFile(filename):
    143  f = open(filename, "r")
    144  content = f.read()
    145  f.close()
    146  return content.replace("\r\n", "\n")
    147 
    148 
    149 def Chunkify(list, chunk_size):
    150  """divides an array into chunks of chunk_size"""
    151  return [list[i:i + chunk_size] for i in range(0, len(list), chunk_size)]
    152 
    153 
    154 def GetText(nodelist):
    155  """Gets the text of from a list of nodes"""
    156  rc = []
    157  for node in nodelist:
    158    if node.nodeType == node.TEXT_NODE:
    159      rc.append(node.data)
    160  return ''.join(rc)
    161 
    162 
    163 def GetElementText(node, name):
    164  """Gets the text of an element"""
    165  elements = node.getElementsByTagName(name)
    166  if len(elements) > 0:
    167    return GetText(elements[0].childNodes)
    168  else:
    169    return None
    170 
    171 
    172 def GetBoolElement(node, name):
    173  text = GetElementText(node, name)
    174  return text.lower() == "true"
    175 
    176 
    177 def GetModel(node):
    178  """Gets the model"""
    179  model = GetElementText(node, "model")
    180  if model and len(model.strip()) == 0:
    181    elements = node.getElementsByTagName("model")
    182    if len(elements) > 0:
    183      model = GetElementText(elements[0], "filename")
    184  return model
    185 
    186 
    187 def RelativizePaths(base, paths, template):
    188  """converts paths to relative paths"""
    189  rels = []
    190  for p in paths:
    191    #print "---"
    192    #print "base: ", os.path.abspath(base)
    193    #print "path: ", os.path.abspath(p)
    194    relpath = os.path.relpath(os.path.abspath(p), os.path.dirname(os.path.abspath(base))).replace("\\", "/")
    195    #print "rel : ", relpath
    196    rels.append(template % relpath)
    197  return "\n".join(rels)
    198 
    199 
    200 def CopyFile(filename, src, dst):
    201  s = os.path.abspath(os.path.join(os.path.dirname(src), filename))
    202  d = os.path.abspath(os.path.join(os.path.dirname(dst), filename))
    203  dst_dir = os.path.dirname(d)
    204  if not os.path.exists(dst_dir):
    205    os.makedirs(dst_dir)
    206  shutil.copyfile(s, d)
    207 
    208 
    209 def CopyShader(filename, src, dst):
    210  s = os.path.abspath(os.path.join(os.path.dirname(src), filename))
    211  d = os.path.abspath(os.path.join(os.path.dirname(dst), filename))
    212  text = ReadFile(s)
    213  # By agreement with the Khronos OpenGL working group we are allowed
    214  # to open source only the .vert and .frag files from the OpenGL ES 2.0
    215  # conformance tests. All other files from the OpenGL ES 2.0 conformance
    216  # tests are not included.
    217  marker = "insert-copyright-here"
    218  new_text = COMMENT_RE.sub(marker, text)
    219  if new_text == text:
    220    print "no matching license found:", s
    221    raise RuntimeError
    222  new_text = REMOVE_COPYRIGHT_RE.sub("", new_text)
    223  glsl_license = '/*\n' + LICENSE + '\n*/'
    224  new_text = new_text.replace(marker, glsl_license)
    225  f = WriteOpen(d)
    226  f.write(new_text)
    227  f.close()
    228 
    229 
    230 def IsOneOf(string, regexs):
    231  for regex in regexs:
    232    if re.match(regex, string):
    233      return True
    234  return False
    235 
    236 
    237 def CheckForUnknownTags(valid_tags, node, depth=1):
    238  """do a hacky check to make sure we're not missing something."""
    239  for child in node.childNodes:
    240    if child.localName and not IsOneOf(child.localName, valid_tags[0]):
    241      print "unsupported tag:", child.localName
    242      print "depth:", depth
    243      raise SyntaxError
    244    else:
    245      if len(valid_tags) > 1:
    246        CheckForUnknownTags(valid_tags[1:], child, depth + 1)
    247 
    248 
    249 def IsFileWeWant(filename):
    250  for f in FILTERS:
    251    if f.search(filename):
    252      return True
    253  return False
    254 
    255 
    256 class TestReader():
    257  """class to read and parse tests"""
    258 
    259  def __init__(self, basepath):
    260    self.tests = []
    261    self.modes = {}
    262    self.patterns = {}
    263    self.basepath = basepath
    264 
    265  def Print(self, msg):
    266    if self.verbose:
    267      print msg
    268 
    269  def MakeOutPath(self, filename):
    270    relpath = os.path.relpath(os.path.abspath(filename), os.path.dirname(os.path.abspath(self.basepath)))
    271    return relpath
    272 
    273  def ReadTests(self, filename):
    274    """reads a .run file and parses."""
    275    Log("reading %s" % filename)
    276    outname = self.MakeOutPath(filename + ".txt")
    277    f = TxtWriter(outname)
    278    dirname = os.path.dirname(filename)
    279    lines = ReadFileAsLines(filename)
    280    count = 0
    281    tests_data = []
    282    for line in lines:
    283      if len(line) > 0 and not line.startswith("#"):
    284        fname = os.path.join(dirname, line)
    285        if line.endswith(".run"):
    286          if self.ReadTests(fname):
    287            f.Write(line + ".txt\n")
    288            count += 1
    289        elif line.endswith(".test"):
    290          tests_data.extend(self.ReadTest(fname))
    291        else:
    292          print "Error in %s:%d:%s" % (filename, count, line)
    293          raise SyntaxError()
    294    if len(tests_data):
    295      global MAX_TESTS_PER_SET
    296      sets = Chunkify(tests_data, MAX_TESTS_PER_SET)
    297      id = 1
    298      for set in sets:
    299        suffix = "_%03d_to_%03d" % (id, id + len(set) - 1)
    300        test_outname = self.MakeOutPath(filename + suffix + ".html")
    301        if os.path.basename(test_outname).startswith("input.run"):
    302          dname = os.path.dirname(test_outname)
    303          folder_name = os.path.basename(dname)
    304          test_outname = os.path.join(dname, folder_name + suffix + ".html")
    305        self.WriteTests(filename, test_outname, {"tests":set})
    306        f.Write(os.path.basename(test_outname) + "\n")
    307        id += len(set)
    308      count += 1
    309    f.Close()
    310    return count
    311 
    312  def ReadTest(self, filename):
    313    """reads a .test file and parses."""
    314    Log("reading %s" % filename)
    315    dom = parse(filename)
    316    tests = dom.getElementsByTagName("test")
    317    tests_data = []
    318    outname = self.MakeOutPath(filename + ".html")
    319    for test in tests:
    320      if not IsFileWeWant(filename):
    321        self.CopyShaders(test, filename, outname)
    322      else:
    323        test_data = self.ProcessTest(test, filename, outname, len(tests_data))
    324        if test_data:
    325          tests_data.append(test_data)
    326    return tests_data
    327 
    328  def ProcessTest(self, test, filename, outname, id):
    329    """Process a test"""
    330    mode = test.getAttribute("mode")
    331    pattern = test.getAttribute("pattern")
    332    self.modes[mode] = 1
    333    self.patterns[pattern] = 1
    334    Log ("%d: mode: %s pattern: %s" % (id, mode, pattern))
    335    method = getattr(self, 'Process_' + pattern)
    336    test_data = method(test, filename, outname)
    337    if test_data:
    338      test_data["pattern"] = pattern
    339    return test_data
    340 
    341  def WriteTests(self, filename, outname, tests_data):
    342    Log("Writing %s" % outname)
    343    template = """<!DOCTYPE html>
    344 <!-- this file is auto-generated. DO NOT EDIT. -->
    345 %(license)s
    346 <html>
    347 <head>
    348 <meta charset="utf-8">
    349 <title>WebGL GLSL conformance test: %(title)s</title>
    350 %(css)s
    351 %(scripts)s
    352 </head>
    353 <body>
    354 <canvas id="example" width="500" height="500" style="width: 16px; height: 16px;"></canvas>
    355 <div id="description"></div>
    356 <div id="console"></div>
    357 </body>
    358 <script>
    359 "use strict";
    360 OpenGLESTestRunner.run(%(tests_data)s);
    361 var successfullyParsed = true;
    362 </script>
    363 </html>
    364 """
    365    css = [
    366      "../../resources/js-test-style.css",
    367      "../../resources/ogles-tests.css",
    368    ]
    369    scripts = [
    370      "../../resources/js-test-pre.js",
    371      "../../resources/webgl-test-utils.js",
    372      "ogles-utils.js",
    373    ]
    374    css_html = RelativizePaths(outname, css, '<link rel="stylesheet" href="%s" />')
    375    scripts_html = RelativizePaths(outname, scripts, '<script src="%s"></script>')
    376 
    377    html_license = '<!--\n' + LICENSE + '\n-->'
    378    f = WriteOpen(outname)
    379    f.write(template % {
    380        "license": html_license,
    381        "css": css_html,
    382        "scripts": scripts_html,
    383        "title": os.path.basename(outname),
    384        "tests_data": json.dumps(tests_data, indent=2)
    385      })
    386    f.close()
    387 
    388 
    389  def CopyShaders(self, test, filename, outname):
    390    """For tests we don't actually support yet, at least copy the shaders"""
    391    shaders = test.getElementsByTagName("shader")
    392    for shader in shaders:
    393      for name in ["vertshader", "fragshader"]:
    394        s = GetElementText(shader, name)
    395        if s and s != "empty":
    396          CopyShader(s, filename, outname)
    397 
    398  #
    399  # pattern handlers.
    400  #
    401 
    402  def Process_compare(self, test, filename, outname):
    403    global MATRIX_RE
    404 
    405    valid_tags = [
    406      ["shader", "model", "glstate"],
    407      ["uniform", "vertshader", "fragshader", "filename", "depthrange"],
    408      ["name", "count", "transpose", "uniform*", "near", "far"],
    409    ]
    410    CheckForUnknownTags(valid_tags, test)
    411 
    412    # parse the test
    413    shaders = test.getElementsByTagName("shader")
    414    shaderInfos = []
    415    for shader in shaders:
    416      v = GetElementText(shader, "vertshader")
    417      f = GetElementText(shader, "fragshader")
    418      CopyShader(v, filename, outname)
    419      CopyShader(f, filename, outname)
    420      info = {
    421        "vertexShader": v,
    422        "fragmentShader": f,
    423      }
    424      shaderInfos.append(info)
    425      uniformElems = shader.getElementsByTagName("uniform")
    426      if len(uniformElems) > 0:
    427        uniforms = {}
    428        info["uniforms"] = uniforms
    429        for uniformElem in uniformElems:
    430          uniform = {"count": 1}
    431          for child in uniformElem.childNodes:
    432            if child.localName == None:
    433              pass
    434            elif child.localName == "name":
    435              uniforms[GetText(child.childNodes)] = uniform
    436            elif child.localName == "count":
    437              uniform["count"] = int(GetText(child.childNodes))
    438            elif child.localName == "transpose":
    439              uniform["transpose"] = (GetText(child.childNodes) == "true")
    440            else:
    441              if "type" in uniform:
    442                print "utype was:", uniform["type"], " found ", child.localName
    443                raise SyntaxError
    444              type_name = GetValidTypeName(child.localName)
    445              uniform["type"] = type_name
    446              valueText = GetText(child.childNodes).replace(",", " ")
    447              uniform["value"] = [float(t) for t in valueText.split()]
    448              m = MATRIX_RE.search(type_name)
    449              if m:
    450                # Why are these backward from the API?!?!?
    451                TransposeMatrix(uniform["value"], int(m.group(1)))
    452    data = {
    453      "name": os.path.basename(outname),
    454      "model": GetModel(test),
    455      "referenceProgram": shaderInfos[1],
    456      "testProgram": shaderInfos[0],
    457    }
    458    gl_states = test.getElementsByTagName("glstate")
    459    if len(gl_states) > 0:
    460      state = {}
    461      data["state"] = state
    462      for gl_state in gl_states:
    463        for state_name in gl_state.childNodes:
    464          if state_name.localName:
    465            values = {}
    466            for field in state_name.childNodes:
    467              if field.localName:
    468                values[field.localName] = GetText(field.childNodes)
    469            state[state_name.localName] = values
    470    return data
    471 
    472  def Process_shaderload(self, test, filename, outname):
    473    """no need for shaderload tests"""
    474    self.CopyShaders(test, filename, outname)
    475 
    476  def Process_extension(self, test, filename, outname):
    477    """no need for extension tests"""
    478    self.CopyShaders(test, filename, outname)
    479 
    480  def Process_createtests(self, test, filename, outname):
    481    Log("createtests Not implemented:  %s" % filename)
    482    self.CopyShaders(test, filename, outname)
    483 
    484  def Process_GL2Test(self, test, filename, outname):
    485    Log("GL2Test Not implemented:  %s" % filename)
    486    self.CopyShaders(test, filename, outname)
    487 
    488  def Process_uniformquery(self, test, filename, outname):
    489    Log("uniformquery Not implemented:  %s" % filename)
    490    self.CopyShaders(test, filename, outname)
    491 
    492  def Process_egl_image_external(self, test, filename, outname):
    493    """no need for egl_image_external tests"""
    494    self.CopyShaders(test, filename, outname)
    495 
    496  def Process_dismount(self, test, filename, outname):
    497    Log("dismount Not implemented:  %s" % filename)
    498    self.CopyShaders(test, filename, outname)
    499 
    500  def Process_build(self, test, filename, outname):
    501    """don't need build tests"""
    502    valid_tags = [
    503      ["shader", "compstat", "linkstat"],
    504      ["vertshader", "fragshader"],
    505    ]
    506    CheckForUnknownTags(valid_tags, test)
    507 
    508    shader = test.getElementsByTagName("shader")
    509    if not shader:
    510      return None
    511    vs = GetElementText(shader[0], "vertshader")
    512    fs = GetElementText(shader[0], "fragshader")
    513    if vs and vs != "empty":
    514      CopyShader(vs, filename, outname)
    515    if fs and fs != "empty":
    516      CopyShader(fs, filename, outname)
    517    data = {
    518      "name": os.path.basename(outname),
    519      "compstat": bool(GetBoolElement(test, "compstat")),
    520      "linkstat": bool(GetBoolElement(test, "linkstat")),
    521      "testProgram": {
    522        "vertexShader": vs,
    523        "fragmentShader": fs,
    524      },
    525    }
    526    attach = test.getElementsByTagName("attach")
    527    if len(attach) > 0:
    528      data["attachError"] = GetElementText(attach[0], "attacherror")
    529    return data
    530 
    531  def Process_coverage(self, test, filename, outname):
    532    Log("coverage Not implemented:  %s" % filename)
    533    self.CopyShaders(test, filename, outname)
    534 
    535  def Process_attributes(self, test, filename, outname):
    536    Log("attributes Not implemented:  %s" % filename)
    537    self.CopyShaders(test, filename, outname)
    538 
    539  def Process_fixed(self, test, filename, outname):
    540    """no need for fixed function tests"""
    541    self.CopyShaders(test, filename, outname)
    542 
    543 
    544 def main(argv):
    545  """This is the main function."""
    546  global VERBOSE
    547 
    548  parser = OptionParser()
    549  parser.add_option(
    550      "-v", "--verbose", action="store_true",
    551      help="prints more output.")
    552 
    553  (options, args) = parser.parse_args(args=argv)
    554 
    555  if len(args) < 1:
    556    pass # fix me
    557 
    558  os.chdir(os.path.dirname(__file__) or '.')
    559 
    560  VERBOSE = options.verbose
    561 
    562  filename = args[0]
    563  test_reader = TestReader(filename)
    564  test_reader.ReadTests(filename)
    565 
    566 
    567 if __name__ == '__main__':
    568  sys.exit(main(sys.argv[1:]))
	tor-browser The Tor Browser
	git clone https://git.dasho.dev/tor-browser.git
	Log \| Files \| Refs \| README \| LICENSE