Skip to content

Commit 36e7759

Browse files
authored
renderdiff: script for updating golden images (#8709)
Adding a python script to enable updating new goldens into a staging branch in the golden repo (filament-assets). The same script can be used in github workflow to automatically create a golden staging branch. This will be useful for users without access to a mac (the only platform for generating goldens as of now).
1 parent 53e28f3 commit 36e7759

File tree

8 files changed

+375
-42
lines changed

8 files changed

+375
-42
lines changed

test/renderdiff/src/golden_manager.py

+80-17
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,79 @@
1+
# Copyright (C) 2025 The Android Open Source Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
115
import os
216
import shutil
17+
import re
318

4-
from utils import execute, ArgParseImpl
19+
from utils import execute, ArgParseImpl, mkdir_p
520

621
GOLDENS_DIR = 'renderdiff'
722

23+
ACCESS_TYPE_TOKEN = 'token'
24+
ACCESS_TYPE_SSH = 'ssh'
25+
ACCESS_TYPE_READ_ONLY = 'read-only'
26+
27+
def _read_git_config(curdir):
28+
with open(os.path.join(curdir, './.git/config'), 'r') as f:
29+
return f.read()
30+
31+
def _write_git_config(curdir, config_str):
32+
with open(os.path.join(curdir, './.git/config'), 'w') as f:
33+
return f.write(config_str)
34+
835
class GoldenManager:
9-
def __init__(self, working_dir, access_token=None):
36+
def __init__(self, working_dir, access_type=ACCESS_TYPE_READ_ONLY, access_token=None):
1037
self.working_dir_ = working_dir
1138
self.access_token_ = access_token
39+
self.access_type_ = access_type
1240
assert os.path.isdir(self.working_dir_),\
1341
f"working directory {self.working_dir_} does not exist"
1442
self._prepare()
1543

1644
def _assets_dir(self):
1745
return os.path.join(self.working_dir_, "filament-assets")
1846

47+
# Returns the directory containing the goldens
48+
def directory(self):
49+
return os.path.join(self._assets_dir(), GOLDENS_DIR)
50+
51+
def _get_repo_url(self):
52+
protocol = ''
53+
protocol_separator = ''
54+
if self.access_type_ == ACCESS_TYPE_SSH:
55+
protocol = 'git@'
56+
protocol_separator = ':'
57+
else:
58+
protocol = 'https://' + \
59+
(f'x-access-token:{self.access_token_}@' if self.access_token_ else '')
60+
protocol_separator = '/'
61+
return f'{protocol}github.com{protocol_separator}google/filament-assets.git'
62+
1963
def _prepare(self):
2064
assets_dir = self._assets_dir()
2165
if not os.path.exists(assets_dir):
22-
access_token_part = ''
23-
if self.access_token_:
24-
access_token_part = f'x-access-token:{self.access_token_}@'
2566
execute(
26-
f'git clone --depth=1 https://{access_token_part}github.com/google/filament-assets.git',
27-
cwd=self.working_dir_)
67+
f'git clone --depth=1 {self._get_repo_url()}',
68+
cwd=self.working_dir_,
69+
capture_output=False
70+
)
2871
else:
72+
if self.access_type_ == ACCESS_TYPE_SSH:
73+
config = _read_git_config(self._assets_dir())
74+
https_url = r'https://github\.com\/google\/filament\.git'
75+
config = re.sub(https_url, self._get_repo_url(), config)
76+
_write_git_config(self._assets_dir(), config)
2977
self.update()
3078

3179
def update(self):
@@ -41,31 +89,46 @@ def merge_to_main(self, branch, push_to_remote=False):
4189
assets_dir = self._assets_dir()
4290
self._git_exec(f'checkout main')
4391
self._git_exec(f'merge --no-ff {branch}')
44-
if push_to_remote and self.access_token_:
92+
if push_to_remote and \
93+
(self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH):
4594
self._git_exec(f'push origin main')
95+
self.update()
4696

47-
def source_from_and_commit(self, src_dir, commit_msg, branch, push_to_remote=False):
97+
def source_from(self, src_dir, commit_msg, branch,
98+
updates=[], deletes=[], push_to_remote=False):
4899
assets_dir = self._assets_dir()
49100
self._git_exec(f'checkout main')
50101
# Force create the branch (note will overwrite the old branch)
51102
self._git_exec(f'switch -C {branch}')
52103
rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
53-
execute(f'rm -rf {rdiff_dir}')
54-
execute(f'mkdir -p {rdiff_dir}')
55-
shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True)
56-
self._git_exec(f'add {GOLDENS_DIR}')
104+
if len(updates) == 0 and len(deletes) == 0:
105+
shutil.rmtree(rdiff_dir, ignore_errors=True)
106+
mkdir_p(rdiff_dir)
107+
shutil.copytree(src_dir, rdiff_dir, dirs_exist_ok=True)
108+
self._git_exec(f'add {GOLDENS_DIR}')
109+
else:
110+
for f in deletes:
111+
self._git_exec(f'remove {os.path.join(GOLDENS_DIR, f)}')
112+
for f in updates:
113+
shutil.copy2(
114+
os.path.join(src_dir, f),
115+
os.path.join(rdiff_dir, f))
116+
self._git_exec(f'add {os.path.join(GOLDENS_DIR, f)}')
57117

58118
TMP_GOLDEN_COMMIT_FILE = '/tmp/golden_commit.txt'
59119

60120
with open(TMP_GOLDEN_COMMIT_FILE, 'w') as f:
61121
f.write(commit_msg)
62-
self._git_exec(f'commit -F {TMP_GOLDEN_COMMIT_FILE}')
63-
if push_to_remote and self.access_token_:
64-
self._git_exec(f'push -f origin ${branch}')
122+
self._git_exec(f'commit -a -F {TMP_GOLDEN_COMMIT_FILE}')
123+
if push_to_remote and \
124+
(self.access_token_ or self.access_type_ == ACCESS_TYPE_SSH):
125+
self._git_exec(f'push -f origin {branch}')
126+
self.update()
65127

66128
def download_to(self, dest_dir, branch='main'):
129+
self._git_exec(f'checkout {branch}')
67130
assets_dir = self._assets_dir()
68-
execute(f'mkdir -p {dest_dir}')
131+
mkdir_p(dest_dir)
69132
rdiff_dir = os.path.join(assets_dir, GOLDENS_DIR)
70133
shutil.copytree(rdiff_dir, dest_dir, dirs_exist_ok=True)
71134

test/renderdiff/src/image_diff.py

+14
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
# Copyright (C) 2025 The Android Open Source Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
115
import tifffile
216
import numpy
317

test/renderdiff/src/run.py

+12-9
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@
1616
import os
1717
import json
1818
import glob
19+
import shutil
1920

20-
from utils import execute, ArgParseImpl
21+
from utils import execute, ArgParseImpl, mkdir_p, mv_f
2122
from parse_test_json import parse_test_config_from_path
2223
from golden_manager import GoldenManager
2324
from image_diff import same_image
@@ -46,7 +47,7 @@ def run_test(gltf_viewer,
4647
assert os.access(gltf_viewer, os.X_OK)
4748

4849
named_output_dir = os.path.join(output_dir, test_config.name)
49-
execute(f'mkdir -p {named_output_dir}')
50+
mkdir_p(named_output_dir)
5051

5152
results = []
5253
for test in test_config.tests:
@@ -82,9 +83,8 @@ def run_test(gltf_viewer,
8283
result = RESULT_OK
8384
out_tif_basename = f'{out_name}.tif'
8485
out_tif_name = f'{named_output_dir}/{out_tif_basename}'
85-
execute(f'mv -f {test.name}0.tif {out_tif_name}', capture_output=False)
86-
execute(f'mv -f {test.name}0.json {named_output_dir}/{test.name}.json',
87-
capture_output=False)
86+
mv_f(f'{test.name}0.tif', out_tif_name)
87+
mv_f(f'{test.name}0.json', f'{named_output_dir}/{test.name}.json')
8888
else:
8989
result = RESULT_FAILED_TO_RENDER
9090
important_print(f'{test_desc} rendering failed with error={out_code}')
@@ -132,11 +132,12 @@ def compare_goldens(render_results, output_dir, goldens):
132132
opengl_lib=args.opengl_lib,
133133
vk_icd=args.vk_icd)
134134

135+
do_compare = False
135136
# The presence of this argument indicates comparison against a set of goldens.
136137
if args.golden_branch:
137138
# prepare goldens working directory
138139
tmp_golden_dir = '/tmp/renderdiff-goldens'
139-
execute(f'mkdir -p {tmp_golden_dir}')
140+
mkdir_p(tmp_golden_dir)
140141

141142
# Download the golden repo into the current working directory
142143
golden_manager = GoldenManager(os.getcwd())
@@ -147,13 +148,15 @@ def compare_goldens(render_results, output_dir, goldens):
147148
glob.glob(f'{os.path.join(tmp_golden_dir, test.name)}/**/*.tif', recursive=True)
148149
}
149150
results = compare_goldens(results, output_dir, goldens)
150-
151+
do_compare = True
151152

152153
with open(f'{output_dir}/results.json', 'w') as f:
153154
f.write(json.dumps(results))
154-
execute(f'cp {args.test} {output_dir}/test.json')
155+
156+
shutil.copy2(args.test, f'{output_dir}/test.json')
155157

156158
failed = [f" {k['name']}" for k in results if k['result'] != RESULT_OK]
157159
success_count = len(results) - len(failed)
158-
important_print(f'Successfully tested {success_count} / {len(results)}' +
160+
op = 'tested' if do_compare else 'rendered'
161+
important_print(f'Successfully {op} {success_count} / {len(results)}' +
159162
('\nFailed:\n' + ('\n'.join(failed)) if len(failed) > 0 else ''))

test/renderdiff/src/update_golden.py

+171
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
# Copyright (C) 2025 The Android Open Source Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import sys
16+
import os
17+
import glob
18+
import time
19+
20+
from golden_manager import GoldenManager, ACCESS_TYPE_SSH, ACCESS_TYPE_TOKEN
21+
from image_diff import same_image
22+
23+
from utils import execute, ArgParseImpl
24+
from utils import prompt_helper, PROMPT_YES, PROMPT_NO
25+
26+
def line_prompt(prompt, validator=lambda a:True):
27+
while True:
28+
res = input(f'{prompt} => ').strip()
29+
if validator(res):
30+
return res
31+
return None
32+
33+
CONFIG_NEW_SRC_DIR = 'goldens_dir'
34+
CONFIG_GOLDENS_BRANCH = 'goldens_branch'
35+
CONFIG_GOLDENS_UPDATES = 'goldens_updates'
36+
CONFIG_GOLDENS_DELETES = 'goldens_deletes'
37+
CONFIG_AUTO_COMMIT = 'auto-commit'
38+
CONFIG_COMMIT_MSG = 'commit_msg'
39+
40+
def _get_current_branch():
41+
code, res = execute('git branch --show-current')
42+
return res.strip()
43+
44+
def _file_as_str(fpath):
45+
with open(fpath, 'r') as f:
46+
return f.read()
47+
48+
def _do_update(golden_manager, config):
49+
deletes = config[CONFIG_GOLDENS_DELETES]
50+
updates = config[CONFIG_GOLDENS_UPDATES]
51+
if len(deletes) == 0 and len(updates) == 0:
52+
print('Nothing to update. Exiting...')
53+
exit(0)
54+
55+
branch = config[CONFIG_GOLDENS_BRANCH]
56+
src_dir = config[CONFIG_NEW_SRC_DIR]
57+
auto_commit = config[CONFIG_AUTO_COMMIT]
58+
commit_msg = config[CONFIG_COMMIT_MSG]
59+
golden_manager.source_from(src_dir, commit_msg, branch,
60+
updates=updates,
61+
deletes=deletes,
62+
push_to_remote=auto_commit)
63+
64+
def _get_deletes_updates(update_dir, golden_dir):
65+
ret_delete = []
66+
ret_update = []
67+
for ext in ['tif', 'json']:
68+
base = set(glob.glob(f'./**/*.{ext}', root_dir=golden_dir, recursive=True))
69+
new = set(glob.glob(f'./**/*.{ext}', root_dir=update_dir, recursive=True))
70+
71+
delete = list(base - new)
72+
update = list(new - base)
73+
74+
for fpath in base.intersection(new):
75+
base_fpath = os.path.join(golden_dir, fpath)
76+
new_fpath = os.path.join(update_dir, fpath)
77+
if (ext == 'tif' and not same_image(new_fpath, base_fpath)) or \
78+
(ext == 'json' and _file_as_str(new_fpath) != _file_as_str(base_fpath)):
79+
update.append(fpath)
80+
81+
ret_update += update
82+
ret_delete += delete
83+
84+
return ret_delete, ret_update
85+
86+
# Ask a bunch of questions to gather the configuration for the update
87+
def _interactive_mode(base_golden_dir):
88+
config = {}
89+
cur_branch = _get_current_branch()
90+
if prompt_helper(
91+
f'Generate the new goldens from your local ' \
92+
f'Filament branch? (branch={cur_branch})') == PROMPT_YES:
93+
code, res = execute('bash ./test/renderdiff/test.sh generate',
94+
capture_output=False)
95+
if code != 0:
96+
print('Failed to generate new goldens')
97+
exit(1)
98+
config[CONFIG_NEW_SRC_DIR] = os.path.join(os.getcwd(), './out/renderdiff_tests/')
99+
else:
100+
def validator(src_dir):
101+
if not os.path.exists(src_dir):
102+
print(f'Cannot find directory {src_dir}. Please try again.')
103+
return False
104+
return True
105+
106+
config[CONFIG_NEW_SRC_DIR] = line_prompt(
107+
'Please provide path of directory containing new goldens',
108+
validator)
109+
110+
if prompt_helper(f'Update new goldens to branch={cur_branch}? '
111+
'(Note that this refers to a branch in the goldens repo, not the Filament repo.)'
112+
) == PROMPT_YES:
113+
config[CONFIG_GOLDENS_BRANCH] = cur_branch
114+
else:
115+
config[CONFIG_GOLDENS_BRANCH] = line_prompt('Please provide new branch name for update')
116+
117+
if prompt_helper(f'Provide a commit message?') == PROMPT_YES:
118+
config[CONFIG_COMMIT_MSG] = line_prompt('Message:')
119+
else:
120+
config[CONFIG_COMMIT_MSG] = f'Update {time.time()} from filament ({cur_branch})'
121+
122+
new_golden_dir = config[CONFIG_NEW_SRC_DIR]
123+
deletes, updates = _get_deletes_updates(new_golden_dir, base_golden_dir)
124+
if len(deletes) + len(updates) != 0:
125+
prompt = 'The following files will be changed:\n' + \
126+
'\n'.join([f' {fname} [delete]' for fname in deletes]) + \
127+
'\n'.join([f' {fname} [update]' for fname in updates]) + \
128+
'\nIs that ok?'
129+
if prompt_helper(prompt) == PROMPT_YES:
130+
config[CONFIG_GOLDENS_DELETES] = deletes
131+
config[CONFIG_GOLDENS_UPDATES] = updates
132+
else:
133+
# We cannot proceed if user answered no.
134+
exit(1)
135+
else:
136+
config[CONFIG_GOLDENS_DELETES] = []
137+
config[CONFIG_GOLDENS_UPDATES] = []
138+
139+
config[CONFIG_AUTO_COMMIT] = \
140+
prompt_helper(f'Commit golden repo changes to remote?') == PROMPT_YES
141+
return config
142+
143+
if __name__ == "__main__":
144+
parser = ArgParseImpl()
145+
parser.add_argument('--branch', help='Branch of the golden repo to write to')
146+
parser.add_argument('--source', help='Directory containing the new goldens')
147+
parser.add_argument('--commit-msg', help='Message for the commit to the golden repo')
148+
parser.add_argument('--golden-repo-token', help='Access token for the golden repo')
149+
150+
args, _ = parser.parse_known_args(sys.argv[1:])
151+
config = {}
152+
golden_manager = GoldenManager(
153+
os.getcwd(),
154+
access_type=ACCESS_TYPE_SSH if not args.golden_repo_token else ACCESS_TYPE_TOKEN,
155+
access_token=args.golden_repo_token
156+
)
157+
base_golden_dir = golden_manager.directory()
158+
if args.branch and args.source and args.commit_msg:
159+
assert os.path.exists(args.source), f'{args.source} (--source) directory not found'
160+
deletes, updates = _get_deletes_updates(args.source, base_golden_dir)
161+
config = {
162+
CONFIG_AUTO_COMMIT: True,
163+
CONFIG_GOLDENS_BRANCH: args.branch,
164+
CONFIG_NEW_SRC_DIR: args.source,
165+
CONFIG_GOLDENS_UPDATES: updates,
166+
CONFIG_GOLDENS_DELETES: deletes,
167+
CONFIG_COMMIT_MSG: args.commit_msg,
168+
}
169+
else:
170+
config = _interactive_mode(base_golden_dir)
171+
_do_update(golden_manager, config)

0 commit comments

Comments
 (0)