runtests

#!./bin/ebb

-- The MIT License (MIT)
-- 
-- Copyright (c) 2015 Stanford University.
-- All rights reserved.
-- 
-- Permission is hereby granted, free of charge, to any person obtaining a
-- copy of this software and associated documentation files (the "Software"),
-- to deal in the Software without restriction, including without limitation
-- the rights to use, copy, modify, merge, publish, distribute, sublicense,
-- and/or sell copies of the Software, and to permit persons to whom the
-- Software is furnished to do so, subject to the following conditions:
-- 
-- The above copyright notice and this permission notice shall be included
-- in all copies or substantial portions of the Software.
-- 
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-- DEALINGS IN THE SOFTWARE.

local ffi = require "ffi"

local USE_EXPERIMENTAL  = false
local USE_MPI           = false
local USE_GPU           = false
local ONE_LINE_ERR      = false

if #arg > 0 then
  for i=1,#arg do
    if arg[i] == '-experimental' or arg[i] == '--experimental' or
       arg[i] == '-exp' or arg[i] == '--exp'
    then
      USE_EXPERIMENTAL = true
    end
    if arg[i] == '-gpu' or arg[i] == '--gpu' then
      USE_GPU = true
    end
    if arg[i] == '-mpi' or arg[i] == '--mpi' then
      USE_MPI = true
    end
    if arg[i] == '-h' or arg[i] == '--help' then
      print("Usage : run_tests [options]")
      print("Options:")
      print("  -h, --help               show this help message and exit")
      print("  -exp, --exp              run tests w/ experimental multinode")
      print("  -gpu, --gpu              run tests on the GPU")
      print("  -mpi, --mpi              run tests with mpirun prefix")
      os.exit(0)
    end
  end
end

local lscmd
if ffi.os == "Windows" then
  lscmd = "cmd /c dir /b /s"
else
  lscmd = "find . | cut -c 3-"
end

local passed     = {}
local bad_passed = {}
local failed     = {}
local disabled   = {}

local exclude = {
  ['tests/test.lua'] = true,
}

local disable_str = '--DISABLE-TEST'
local disable_gpu_str = '--DISABLE-ON-GPU'
local disable_dist_str = '--DISABLE-DISTRIBUTED'
local function str_starts_with(str, prefix)
  return string.sub(str,1,#prefix) == prefix
end
local function is_disabled (filename)
  local h = io.open(filename, "r")
  local line1 = h:read()
  local line2 = h:read()
  io.close(h)
  local disabled_all    = false
  local disabled_gpu    = false
  local disabled_dist   = false
  if line1 then
    disabled_all    = disabled_all or str_starts_with(line1, disable_str)
    disabled_gpu    = disabled_gpu or str_starts_with(line1, disable_gpu_str)
    disabled_dist   = disabled_dist or
                      str_starts_with(line1, disable_dist_str)
  end
  if line2 then
    disabled_all    = disabled_all or str_starts_with(line2, disable_str)
    disabled_gpu    = disabled_gpu or str_starts_with(line2, disable_gpu_str)
    disabled_dist   = disabled_dist or
                      str_starts_with(line2, disable_dist_str)
  end
  return disabled_all or (USE_GPU     and disabled_gpu)
                      or (USE_EXPERIMENTAL and disabled_dist)
end

local function output_name (filename)
  local outname = filename:gsub("/(.-)%.t$", "/%1.out")
  -- check whether the file exists
  if outname ~= filename then
    local f = io.open(outname,"r")
    if f then
      io.close(f)
      return outname
    end
  end
  -- implicitly return nil if there is no file match
end

print("==================")
print("= Running tests...")
print("==================")
for line in io.popen(lscmd):lines() do
  if ffi.os == "Windows" then
    local cwd = io.popen("cmd /c echo %cd%"):read()
    line = line:sub(cwd:len()+2)
    line = line:gsub("\\","/")
  end
  local file = line:match("^(tests/.*%.t)$") or line:match("^(tests/.*%.lua)$")
  local out_file = file and output_name(file)
  if file and not exclude[file] then
    if is_disabled(file) then
      table.insert(disabled, file)
    else
      print(file)
      local should_fail = (file:match("fails/") ~= nil)
      local execstring = "./ebb "
      if USE_EXPERIMENTAL then
        execstring = execstring .. " -n 5 "
      end
      if USE_GPU then
        execstring = execstring .. " --gpu "
      end
      execstring = execstring .. file
      -- If we expect output from this test, log stdout
      if out_file then
        execstring = execstring .. " | grep -v INFO > .test_out"
      elseif should_fail then
        execstring = execstring .. " > /dev/null 2>&1"
      end

      if USE_MPI then
        execstring = 'mpirun -n 2 -H n0000,n0001 -npernode 1 -bind-to none '..execstring
      end

      --things in the fail directory should cause terra compiler errors
      --we dont check for the particular error
      local success = os.execute(execstring)
      -- if we expect output, modulate the success appropriately
      if out_file and success == 0 then
        -- compare .test_out to out_file
        local diff_string = 'diff .test_out ' .. out_file
        success = os.execute(diff_string)
      end
      -- record/report failure/success appropriately
      if success ~= 0 and not should_fail then
        table.insert(failed,file)
        print(file .. " \27[31mFAILED\27[0m")
      elseif success == 0 and should_fail then
        table.insert(bad_passed,file)
        print(file .. " \27[31mFAILED\27[0m")
      else
        table.insert(passed,file)
      end
    end
  end
end

-- test whether the coverageanalysis exists
local coverage_on = os.execute('test -f coverageinfo.lua') == 0
if coverage_on then
  print('-- Assembling Coverage Analysis Report --')
  os.execute('./covanalysis')
  os.execute('rm coverageinfo.lua')
end

-- cleanup temp files if they exist
os.execute('rm .test_out')

print("==================")
print()

local function printtests(nm,lst)
  if #lst > 0 then
    print("==================")
    print("= "..nm)
    print("==================")
    for i,e in ipairs(lst) do
      print(e)
    end
    print("==================")
    print()
  end
end
--printtests("passing tests",passed)
printtests("passing tests", passed)
printtests("FAILING tests",failed)
printtests("passed but should have failed",bad_passed)
printtests("disabled tests",disabled)

print(tostring(#passed).." tests passed, "..tostring(#failed + #bad_passed).." tests failed. " .. tostring(#disabled) .. " tests disabled.")

-- make failures visible to other testing infrastructure
os.exit(#failed + #bad_passed)