framework/replay: adapt GFXReconstruct backend to >=0.9.4

The info tool is now spitting the total frames in the 3rd line.

Also, we can now get rid of the VK_LAYER_LUNARG_screenshot layer and
use replay's screenshot built-in flag instead.

v2:
  - Check the last frame through a regexp (Martin).
  - Check replay's minimum supported version (Martin).

Signed-off-by: Andres Gomez <agomez@igalia.com>
Reviewed-by: Martin Peres <martin.peres@mupuf.org>
Reviewed-by: Juan A. Suarez <jasuarez@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/piglit/-/merge_requests/473>

6 files changed, 212 insertions, 124 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3b7138fc..72c74219f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@
 # repository's registry will be used there as well.
 variables:
   UPSTREAM_REPO: mesa/piglit
-  DEBIAN_TAG: "2020-10-16"
+  DEBIAN_TAG: "2021-02-09"
   DEBIAN_VERSION: buster-slim
   DEBIAN_IMAGE: "$CI_REGISTRY_IMAGE/debian/$DEBIAN_VERSION:$DEBIAN_TAG"
   WINDOWS_TAG: "2020-08-18"
diff --git a/.gitlab-ci/debian-install.sh b/.gitlab-ci/debian-install.sh
index cc52710bc..b5a6f6d21 100644
--- a/.gitlab-ci/debian-install.sh
+++ b/.gitlab-ci/debian-install.sh
@@ -48,6 +48,7 @@ apt-get install -y \
   python3-mako \
   python3-mock \
   python3-numpy \
+  python3-packaging \
   python3-pil \
   python3-pip \
   python3-psutil \
diff --git a/framework/replay/backends/abstract.py b/framework/replay/backends/abstract.py
index 905549b8c..f84a4e0e3 100644
--- a/framework/replay/backends/abstract.py
+++ b/framework/replay/backends/abstract.py
@@ -95,7 +95,7 @@ class DumpBackend(metaclass=abc.ABCMeta):
         print(logoutput.decode(errors='replace'))
         if ret.returncode:
             raise RuntimeError(
-                '[dump_traces_images] Process failed with error code: {}'.format(
+                '[dump_trace_images] Process failed with error code: {}'.format(
                     ret.returncode))
 
 
diff --git a/framework/replay/backends/gfxreconstruct.py b/framework/replay/backends/gfxreconstruct.py
index 07cb3b846..b247a5b67 100644
--- a/framework/replay/backends/gfxreconstruct.py
+++ b/framework/replay/backends/gfxreconstruct.py
@@ -28,11 +28,14 @@
 """ Module providing a GFXReconstruct dump backend for replayer """
 
 import os
+import re
 import subprocess
 
+from packaging import version
 from os import path
 
 from framework import core, exceptions
+from . import DumpBackendError
 from .abstract import DumpBackend, dump_handler
 from .register import Registry
 
@@ -43,6 +46,11 @@ __all__ = [
 ]
 
 
+_MIN_VERSION = version.Version('0.9.4')
+_VERSION_RE = re.compile('\s*GFXReconstruct Version\s*([0-9]\.[0-9]\.[0-9])')
+
+_TOTAL_FRAMES_RE = re.compile('\s*Total frames:\s*([0-9]*)')
+
 class GFXReconstructBackend(DumpBackend):
     """ replayer's GFXReconstruct dump backend
 
@@ -74,44 +82,59 @@ class GFXReconstructBackend(DumpBackend):
         ret = subprocess.run(cmd, stdout=subprocess.PIPE)
         lines = ret.stdout.decode(errors='replace').splitlines()
         print(ret.stdout.decode(errors='replace'))
-        if lines:
-            c = lines[0].split(': ', 1)
-            if len(c) > 1 and c[1].isnumeric():
-                return int(c[1])
-        return -1
+        try:
+            frames = re.search(_TOTAL_FRAMES_RE, lines[2])
+            return int(frames.group(1))
+        except:
+            return -1
+
+    def _check_version(self, gfxrecon_replay_bin):
+        cmd = [gfxrecon_replay_bin, '--version']
+        ret = subprocess.run(cmd, stdout=subprocess.PIPE)
+        lines = ret.stdout.decode(errors='replace').splitlines()
+        print(ret.stdout.decode(errors='replace'))
+        try:
+            v = re.search(_VERSION_RE, lines[1])
+            current = version.Version(v.group(1))
+        except:
+            raise DumpBackendError(
+                '[dump_trace_images] Unable to check the current '
+                'gfxrecon-replay version.')
+        if _MIN_VERSION > current:
+            raise DumpBackendError(
+                '[dump_trace_images] The current gfxrecon-replay version '
+                'is {}. Try to update, at least to the {} version.'.format(
+                    current, _MIN_VERSION))
 
     @dump_handler
     def dump(self):
         from PIL import Image
         outputprefix = path.join(self._output_dir,
                                  path.basename(self._trace_path))
-        if not self._calls:
-            # FIXME: The VK_LAYER_LUNARG_screenshot numbers the calls from 0 to
-            # (total-num-calls - 1) while gfxreconstruct does it from 1 to
-            # total-num-calls:
-            # https://github.com/LunarG/gfxreconstruct/issues/284
-            self._calls = [str(max(-1, self._get_last_frame_call() - 1))]
         gfxrecon_replay_bin = core.get_option(
             'PIGLIT_REPLAY_GFXRECON_REPLAY_BINARY',
             ('replay', 'gfxrecon-replay_bin'),
             default='gfxrecon-replay')
+        self._check_version(gfxrecon_replay_bin)
+        if not self._calls:
+            self._calls = [str(self._get_last_frame_call())]
         gfxrecon_replay_extra_args = core.get_option(
             'PIGLIT_REPLAY_GFXRECON_REPLAY_EXTRA_ARGS',
             ('replay', 'gfxrecon-replay_extra_args'),
             default='').split()
-        cmd = ([gfxrecon_replay_bin]
-               + gfxrecon_replay_extra_args + [self._trace_path])
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = ','.join(self._calls)
-        env['VK_SCREENSHOT_DIR'] = self._output_dir
-        self._run_logged_command(cmd, env)
+        cmd = ([gfxrecon_replay_bin] + gfxrecon_replay_extra_args +
+               ['--screenshots', ','.join(self._calls),
+                '--screenshot-dir', self._output_dir,
+                self._trace_path])
+        self._run_logged_command(cmd, None)
         for c in self._calls:
-            ppm = '{}.ppm'.format(path.join(self._output_dir, c))
+            bmp = '{}_frame_{}.bmp'.format(path.join(self._output_dir,
+                                                    'screenshot'),
+                                          c)
             outputfile = '{}-{}.png'.format(outputprefix, c)
-            print('Writing: {} to {}'.format(ppm, outputfile))
-            Image.open(ppm).save(outputfile)
-            os.remove(ppm)
+            print('Writing: {} to {}'.format(bmp, outputfile))
+            Image.open(bmp).save(outputfile)
+            os.remove(bmp)
 
 
 REGISTRY = Registry(
diff --git a/tox.ini b/tox.ini
index 67e688f84..fb84d0f7a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,7 @@ deps =
     functional: pytest>=3.9
     pytest-mock==1.11.2
     {accel,noaccel}: requests-mock
+    {accel,noaccel}: packaging
     pytest-pythonpath
     pytest-raises
     pytest-timeout==1.2.1
diff --git a/unittests/framework/replay/backends/test_gfxreconstruct.py b/unittests/framework/replay/backends/test_gfxreconstruct.py
index ae9b92dae..7f56e4191 100644
--- a/unittests/framework/replay/backends/test_gfxreconstruct.py
+++ b/unittests/framework/replay/backends/test_gfxreconstruct.py
@@ -56,56 +56,73 @@ class TestGFXReconstructBackend(object):
             if cmd[-1] == self.vk_last_frame_fails_trace_path:
                 ret.stdout = b''
             else:
-                ret.stdout = bytearray('Total frames: ' +
-                                       str(int(self.vk_trace_last_call) + 1) +
-                                       textwrap.dedent('''
-                                       Application info:
-                                               Application name: vkcube
-                                               Application version: 0
-                                               Engine name: vkcube
-                                               Engine version: 0
-                                               Target API version: 4194304 (1.0.0)
-
-                                       Physical device info:
-                                               Device name: TESTING DEVICE
-                                               Device ID: 0xabcd
-                                               Vendor ID: 0xefgh
-                                               Driver version: 83890275 (0x5001063)
-                                               API version: 4202627 (1.2.131)
-
-                                       Device memory allocation info:
-                                               Total allocations: 5
-                                               Min allocation size: 1216
-                                               Max allocation size: 540680
-
-                                       Pipeline info:
-                                               Total graphics pipelines: 1
-                                               Total compute pipelines: 0
-                                       '''),
-                                       'utf-8')
+                info_output = '''\
+                File info:
+                        Compression format: Zstandard
+                        Total frames: {}
+
+                Application info:
+                        Application name: vkcube
+                        Application version: 0
+                        Engine name: vkcube
+                        Engine version: 0
+                        Target API version: 4194304 (1.0.0)
+
+                Physical device info:
+                        Device name: TESTING DEVICE
+                        Device ID: 0xabcd
+                        Vendor ID: 0xefgh
+                        Driver version: 83890275 (0x5001063)
+                        API version: 4202627 (1.2.131)
+
+                Device memory allocation info:
+                        Total allocations: 5
+                        Min allocation size: 1216
+                        Max allocation size: 540680
+
+                Pipeline info:
+                        Total graphics pipelines: 1
+                        Total compute pipelines: 0
+                '''.format(int(self.vk_trace_last_call))
+                ret.stdout = bytearray(textwrap.dedent(info_output), 'utf-8')
+
+            return ret
+        elif cmd[-1] == '--version':
+            # VK replay --version
+            ret = subprocess.CompletedProcess(cmd, 0)
+            if cmd[0] == self.gfxrecon_replay_bogus:
+                ret.stdout = b''
+            else:
+                if cmd[0] == self.gfxrecon_replay_old:
+                    version = '0.9.3'
+                else:
+                    version = '0.9.4'
+                version_output = '''\
+                gfxrecon-replay version info:
+                  GFXReconstruct Version {} (v{}:3738dec)
+                  Vulkan Header Version 1.2.162
+                '''.format(version, version)
+                ret.stdout = bytearray(textwrap.dedent(version_output), 'utf-8')
 
             return ret
         elif cmd[0].endswith(self.gfxrecon_replay):
             # VK replay
             ret = subprocess.CompletedProcess(cmd, 0)
             if cmd[-1] != self.vk_replay_crashes_trace_path:
-                calls = env['VK_SCREENSHOT_FRAMES']
-                prefix = env['VK_SCREENSHOT_DIR']
+                calls = cmd[-4]
+                prefix = cmd[-2]
                 ret.stdout = b''
-                if env['VK_INSTANCE_LAYERS'] == 'VK_LAYER_LUNARG_screenshot':
-                    for call in calls.split(','):
-                        if (call != self.vk_trace_wrong_call and
-                            call != '-1'):
-                            from PIL import Image
-                            dump = path.join(prefix, call + '.ppm')
-                            rgba = 'ff00ffff'
-                            color = [int(rgba[0:2], 16), int(rgba[2:4], 16),
-                                     int(rgba[4:6], 16), int(rgba[6:8], 16)]
-                            Image.frombytes('RGBA', (32, 32),
-                                            bytes(color * 32 * 32)).save(dump)
-                            ret.stdout += bytearray('Screen Capture file '
-                                                    'is: ' + dump + '\n',
-                                                    'utf-8')
+                for call in calls.split(','):
+                    if (call != self.vk_trace_wrong_call and
+                        call != '-1'):
+                        from PIL import Image
+                        dump = path.join(
+                            prefix, 'screenshot_frame_{}.bmp'.format(call))
+                        rgba = 'ff00ffff'
+                        color = [int(rgba[0:2], 16), int(rgba[2:4], 16),
+                                 int(rgba[4:6], 16), int(rgba[6:8], 16)]
+                        Image.frombytes('RGBA', (32, 32),
+                                        bytes(color * 32 * 32)).save(dump)
                 ret.stdout += bytearray('35.650065 fps, 0.280504 seconds, '
                                         '10 frames, 1 loop, framerange 1-10\n',
                                         'utf-8')
@@ -128,6 +145,8 @@ class TestGFXReconstructBackend(object):
         OPTIONS.device_name = 'test-device'
         self.gfxrecon_info = 'gfxrecon-info'
         self.gfxrecon_replay = 'gfxrecon-replay'
+        self.gfxrecon_replay_old = '/old/gfxrecon-replay'
+        self.gfxrecon_replay_bogus = '/bogus/gfxrecon-replay'
         self.gfxrecon_replay_extra = ''
         self.vk_trace_path = tmpdir.mkdir(
             'db-path').join('KhronosGroup-Vulkan-Tools/vkcube.gfxr').strpath
@@ -137,9 +156,9 @@ class TestGFXReconstructBackend(object):
         self.vk_replay_crashes_trace_path = tmpdir.join(
             'db-path',
             'replay/fails.gfxr').strpath
-        self.vk_trace_calls = '2,5'
-        self.vk_trace_last_call = '9'
-        self.vk_trace_wrong_call = '10'
+        self.vk_trace_calls = '3,6'
+        self.vk_trace_last_call = '10'
+        self.vk_trace_wrong_call = '11'
         self.output_dir = tmpdir.mkdir('results').strpath
         self.results_partial_path = path.join('trace', OPTIONS.device_name)
         self.m_gfxreconstruct_subprocess_run = mocker.patch(
@@ -190,17 +209,19 @@ class TestGFXReconstructBackend(object):
         test = backends.gfxreconstruct.GFXReconstructBackend(trace_path)
         assert test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
         m_calls = [self.mocker.call(
-            [self.gfxrecon_info, trace_path],
+            [self.gfxrecon_replay, '--version'],
             stdout=subprocess.PIPE),
                    self.mocker.call(
-                       [self.gfxrecon_replay, trace_path],
-                       env=env, stdout=subprocess.PIPE)]
-        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+                       [self.gfxrecon_info, trace_path],
+                       stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 3
         self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert path.exists(snapshot_prefix + call + '.png')
@@ -230,18 +251,20 @@ class TestGFXReconstructBackend(object):
         test = backends.gfxreconstruct.GFXReconstructBackend(trace_path)
         assert test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
         m_calls = [self.mocker.call(
-            [self.gfxrecon_info, trace_path],
+            [self.gfxrecon_replay, '--version'],
             stdout=subprocess.PIPE),
                    self.mocker.call(
+                       [self.gfxrecon_info, trace_path],
+                       stdout=subprocess.PIPE),
+                   self.mocker.call(
                        [self.gfxrecon_replay] +
-                       gfxrecon_replay_extra.split() + [trace_path],
-                       env=env, stdout=subprocess.PIPE)]
-        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+                       gfxrecon_replay_extra.split() +
+                       ['--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 3
         self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert path.exists(snapshot_prefix + call + '.png')
@@ -260,17 +283,19 @@ class TestGFXReconstructBackend(object):
         assert test.dump()
         snapshot_prefix = path.join(self.output_dir,
                                     path.basename(trace_path) + '-')
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = self.output_dir
         m_calls = [self.mocker.call(
-            [self.gfxrecon_info, trace_path],
+            [self.gfxrecon_replay, '--version'],
             stdout=subprocess.PIPE),
                    self.mocker.call(
-                       [self.gfxrecon_replay, trace_path],
-                       env=env, stdout=subprocess.PIPE)]
-        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+                       [self.gfxrecon_info, trace_path],
+                       stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', self.output_dir,
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 3
         self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert path.exists(snapshot_prefix + call + '.png')
@@ -288,13 +313,17 @@ class TestGFXReconstructBackend(object):
             trace_path, calls=calls.split(','))
         assert test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
-        self.m_gfxreconstruct_subprocess_run.assert_called_once_with(
-            [self.gfxrecon_replay, trace_path],
-            env=env, stdout=subprocess.PIPE)
+        m_calls = [self.mocker.call(
+            [self.gfxrecon_replay, '--version'],
+            stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+        self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert path.exists(snapshot_prefix + call + '.png')
 
@@ -311,13 +340,17 @@ class TestGFXReconstructBackend(object):
             trace_path, calls=calls.split(','))
         assert not test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
-        self.m_gfxreconstruct_subprocess_run.assert_called_once_with(
-            [self.gfxrecon_replay, trace_path],
-            env=env, stdout=subprocess.PIPE)
+        m_calls = [self.mocker.call(
+            [self.gfxrecon_replay, '--version'],
+            stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+        self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert not path.exists(snapshot_prefix + call + '.png')
 
@@ -332,17 +365,19 @@ class TestGFXReconstructBackend(object):
         test = backends.gfxreconstruct.GFXReconstructBackend(trace_path)
         assert not test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
         m_calls = [self.mocker.call(
-            [self.gfxrecon_info, trace_path],
+            [self.gfxrecon_replay, '--version'],
             stdout=subprocess.PIPE),
                    self.mocker.call(
-                       [self.gfxrecon_replay, trace_path],
-                       env=env, stdout=subprocess.PIPE)]
-        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+                       [self.gfxrecon_info, trace_path],
+                       stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 3
         self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert not path.exists(snapshot_prefix + call + '.png')
@@ -358,17 +393,45 @@ class TestGFXReconstructBackend(object):
         test = backends.gfxreconstruct.GFXReconstructBackend(trace_path)
         assert not test.dump()
         snapshot_prefix = trace_path + '-'
-        env = os.environ.copy()
-        env['VK_INSTANCE_LAYERS'] = 'VK_LAYER_LUNARG_screenshot'
-        env['VK_SCREENSHOT_FRAMES'] = calls
-        env['VK_SCREENSHOT_DIR'] = path.dirname(trace_path)
         m_calls = [self.mocker.call(
-            [self.gfxrecon_info, trace_path],
+            [self.gfxrecon_replay, '--version'],
             stdout=subprocess.PIPE),
                    self.mocker.call(
-                       [self.gfxrecon_replay, trace_path],
-                       env=env, stdout=subprocess.PIPE)]
-        assert self.m_gfxreconstruct_subprocess_run.call_count == 2
+                       [self.gfxrecon_info, trace_path],
+                       stdout=subprocess.PIPE),
+                   self.mocker.call(
+                       [self.gfxrecon_replay,
+                        '--screenshots', calls,
+                        '--screenshot-dir', path.dirname(trace_path),
+                        trace_path],
+                       env=None, stdout=subprocess.PIPE)]
+        assert self.m_gfxreconstruct_subprocess_run.call_count == 3
         self.m_gfxreconstruct_subprocess_run.assert_has_calls(m_calls)
         for call in calls.split(','):
             assert not path.exists(snapshot_prefix + call + '.png')
+
+    @pytest.mark.parametrize('option', [
+        (0),
+        (1),
+    ])
+    def test_dump_vk_replay_invalid(self, option, config):
+        """Tests for the dump method: replay's version is invalid.
+
+        Check a basic VK dump. Checking replay's version tells us it's invalid.
+
+        """
+        if option == 0:
+            self.gfxrecon_replay = self.gfxrecon_replay_old
+        elif option == 1:
+            self.gfxrecon_replay = self.gfxrecon_replay_bogus
+        config.set('replay', 'gfxrecon-replay_bin', self.gfxrecon_replay)
+        calls = self.vk_trace_last_call
+        trace_path = self.vk_trace_path
+        test = backends.gfxreconstruct.GFXReconstructBackend(trace_path)
+        assert not test.dump()
+        snapshot_prefix = trace_path + '-'
+        self.m_gfxreconstruct_subprocess_run.assert_called_once_with(
+            [self.gfxrecon_replay, '--version'],
+            stdout=subprocess.PIPE)
+        for call in calls.split(','):
+            assert not path.exists(snapshot_prefix + call + '.png')