Skip to content

Commit a324080

Browse files
KevinJWKevinJW
committed
Add support for environment variables in minimal configuration archives
If archiving a minimal config and environment variables are used, it must be possible for all the needed variables to be resolved and any files pointed to should also exist at the time of archiving. The values of the environment will be written into the saved configuration in the environment section of the configuration file. Ths configuration stored thus may be different to the initial config being passed for archiving. Needs tests adding and futher discussion in the TSC meetings. Signed-off-by: Kevin Wheatley <[email protected]>
1 parent f9a5f03 commit a324080

File tree

5 files changed

+99
-54
lines changed

5 files changed

+99
-54
lines changed

‎include/OpenColorIO/OpenColorIO.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1507,7 +1507,7 @@ class OCIOEXPORT Config
15071507
*
15081508
* \return bool Archivable if true.
15091509
*/
1510-
bool isArchivable() const;
1510+
bool isArchivable(bool minimal) const;
15111511

15121512
/**
15131513
* \brief Archive the config and its LUTs into the specified output stream.

‎src/OpenColorIO/Config.cpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5537,7 +5537,7 @@ ConfigIOProxyRcPtr Config::getConfigIOProxy() const
55375537
return getImpl()->m_context->getConfigIOProxy();
55385538
}
55395539

5540-
bool Config::isArchivable() const
5540+
bool Config::isArchivable(bool minimal) const
55415541
{
55425542
ConstContextRcPtr context = getCurrentContext();
55435543

@@ -5550,7 +5550,7 @@ bool Config::isArchivable() const
55505550
}
55515551

55525552
// Utility lambda to check the following criteria.
5553-
auto validatePathForArchiving = [](const std::string & path)
5553+
auto validatePathForArchiving = [&minimal](const std::string & path)
55545554
{
55555555
// Using the normalized path.
55565556
const std::string normPath = pystring::os::path::normpath(path);
@@ -5561,8 +5561,9 @@ bool Config::isArchivable() const
55615561
pystring::startswith(normPath, "..") ||
55625562
// 3) A context variable may not be located at the start of the path.
55635563
(ContainsContextVariables(path) && //TODO: if we can resolve context, do so and validate path to file
5564-
(StringUtils::Find(path, "$") == 0 ||
5565-
StringUtils::Find(path, "%") == 0)))
5564+
(!minimal &&
5565+
(StringUtils::Find(path, "$") == 0 ||
5566+
StringUtils::Find(path, "%") == 0))))
55665567
{
55675568
return false;
55685569
}

‎src/OpenColorIO/OCIOZArchive.cpp‎

Lines changed: 70 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include "Platform.h"
1717
#include "utils/StringUtils.h"
1818
#include "transforms/FileTransform.h"
19+
#include "Logging.h"
1920

2021
#include "OCIOZArchive.h"
2122

@@ -183,6 +184,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
183184
if (!possibleFormats.empty())
184185
{
185186
// The extension is supported. Add the current file to the OCIOZ archive.
187+
std::ostringstream logMessage;
188+
logMessage << "Adding file: " << absPath;
189+
LogInfo(logMessage.str());
186190
if (mz_zip_writer_add_path(
187191
archiver, absPath.c_str(),
188192
configWorkingDirectory, 0, 1) != MZ_OK)
@@ -201,9 +205,9 @@ void addSupportedFiles(void * archiver, const char * path, const char * configWo
201205
}
202206
}
203207

204-
void addReferencedFiles(void * archiver, const Config & config)
208+
ContextRcPtr addReferencedFiles(void * archiver, const ConfigRcPtr & config)
205209
{
206-
ConstContextRcPtr context = config.getCurrentContext();
210+
ConstContextRcPtr context = config->getCurrentContext();
207211
ContextRcPtr ctxFilepath = Context::Create();
208212
ctxFilepath->setSearchPath(context->getSearchPath());
209213
ctxFilepath->setWorkingDir(context->getWorkingDir());
@@ -212,20 +216,42 @@ void addReferencedFiles(void * archiver, const Config & config)
212216
auto prefixLength = std::string(context->getWorkingDir()).length() + 1; // +1 add trailing '/' TODO: improve this
213217

214218
std::set<std::string> files;
215-
config.GetAllFileReferences(files);
219+
config->GetAllFileReferences(files);
220+
std::set<std::string> added_files;
216221
for (const auto &file : files)
217222
{
218223
const std::string resolvedPath = context->resolveFileLocation(file.c_str(), ctxFilepath);
219224
const std::string relativePath = resolvedPath.substr(prefixLength);
225+
std::ostringstream logMessage;
220226

221-
auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
222-
if (returnCode != MZ_OK)
227+
if (added_files.find(relativePath) == added_files.end())
223228
{
224-
std::ostringstream os;
225-
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
226-
throw Exception(os.str().c_str());
229+
logMessage << "Adding file: " << file << " -> " << relativePath;
230+
LogInfo(logMessage.str());
231+
auto returnCode = mz_zip_writer_add_file(archiver, resolvedPath.c_str(), relativePath.c_str());
232+
if (returnCode != MZ_OK)
233+
{
234+
std::ostringstream os;
235+
os << "Could not write file " << resolvedPath << " to in-memory archive.()" << returnCode << ")";
236+
throw Exception(os.str().c_str());
237+
}
227238
}
239+
else
240+
{
241+
logMessage << "Skipping already added file: " << file << " -> " << relativePath;
242+
LogInfo(logMessage.str());
243+
}
244+
added_files.insert(relativePath);
228245
}
246+
247+
const auto variableCount = ctxFilepath->getNumStringVars();
248+
for (auto i = 0; i != variableCount; ++i)
249+
{
250+
std::ostringstream logMessage;
251+
logMessage << "Used Variable: " << ctxFilepath->getStringVarNameByIndex(i) << " -> " << ctxFilepath->getStringVarByIndex(i);
252+
LogInfo(logMessage.str());
253+
}
254+
return ctxFilepath;
229255
}
230256

231257
//////////////////////////////////////////////////////////////////////////////////////
@@ -250,8 +276,9 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
250276
mz_zip_file file_info;
251277

252278
flags = EnvironmentOverride(flags);
279+
const bool minimal = HasFlag(flags, ARCHIVE_FLAGS_MINIMAL);
253280

254-
if (!config.isArchivable()) // TODO: pass in flags?
281+
if (!config.isArchivable(minimal)) // TODO: pass in flags?
255282
{
256283
std::ostringstream os;
257284
os << "Config is not archivable.";
@@ -261,11 +288,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
261288
// Initialize.
262289
memset(&file_info, 0, sizeof(file_info));
263290

264-
// Retrieve and store the config as string.
265-
std::stringstream ss;
266-
config.serialize(ss);
267-
std::string configStr = ss.str();
268-
269291
// Write zip to memory stream.
270292
#if MZ_VERSION_BUILD >= 040000
271293
write_mem_stream = mz_stream_mem_create();
@@ -303,10 +325,38 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
303325
// Open the in-memory zip.
304326
if (mz_zip_writer_open(archiver, write_mem_stream, 0) == MZ_OK)
305327
{
328+
ConfigRcPtr editiableConfig = config.createEditableCopy(); // TODO: is this a little heavy handed just so we can modify the envronment?
329+
///////////////////////
330+
// Adding LUT files //
331+
///////////////////////
332+
if (minimal)
333+
{
334+
ContextRcPtr archivedContext = addReferencedFiles(archiver, editiableConfig);
335+
336+
// Need to make sure the used evironment context is in the config
337+
const auto variableCount = archivedContext->getNumStringVars();
338+
for (auto i = 0; i != variableCount; ++i)
339+
{
340+
editiableConfig->addEnvironmentVar(archivedContext->getStringVarNameByIndex(i), archivedContext->getStringVarByIndex(i));
341+
}
342+
}
343+
else
344+
{
345+
// Add all supported files to in-memory zip from any directories under working directory.
346+
// (recursive)
347+
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);
348+
}
349+
350+
//////////////////////////////
351+
// Adding config to archive //
352+
//////////////////////////////
306353
// Use a hardcoded name for the config's filename inside the archive.
307354
std::string configFullname = std::string(OCIO_CONFIG_DEFAULT_NAME) +
308355
std::string(OCIO_CONFIG_DEFAULT_FILE_EXT);
309356

357+
std::stringstream ss;
358+
editiableConfig->serialize(ss);
359+
std::string configStr = ss.str();
310360
// Get config string size.
311361
int32_t configSize = (int32_t) configStr.size();
312362

@@ -316,11 +366,13 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
316366
file_info.version_madeby = MZ_VERSION_MADEBY;
317367
file_info.compression_method = MZ_COMPRESS_METHOD_DEFLATE;
318368
file_info.flag = MZ_ZIP_FLAG_UTF8;
319-
file_info.uncompressed_size = configSize;
369+
file_info.uncompressed_size = configSize; // Retrieve and store the config as string.
370+
371+
constexpr uint32_t posix_attrib = 0100644; // File with -rw-r--r-- permissions
372+
constexpr uint32_t msdos_attrib = 0200; // MSDOS equivalent
373+
file_info.external_fa = msdos_attrib;
374+
file_info.external_fa |= (posix_attrib << 16);
320375

321-
//////////////////////////////
322-
// Adding config to archive //
323-
//////////////////////////////
324376
int32_t written = 0;
325377
// Opens an entry in the in-memory zip file for writing.
326378
if (mz_zip_writer_entry_open(archiver, &file_info) == MZ_OK)
@@ -343,16 +395,6 @@ void archiveConfig(std::ostream & ostream, const Config & config, const char * c
343395
throw Exception(os.str().c_str());
344396
}
345397

346-
///////////////////////
347-
// Adding LUT files //
348-
///////////////////////
349-
// Add all supported files to in-memory zip from any directories under working directory.
350-
// (recursive)
351-
if (HasFlag(flags, ARCHIVE_FLAGS_MINIMAL))
352-
addReferencedFiles(archiver, config);
353-
else
354-
addSupportedFiles(archiver, configWorkingDirectory, configWorkingDirectory);
355-
356398
std::ostringstream comment;
357399
comment << "Configuration written by archiveConfig() OCIO: " << GetVersion();
358400
mz_zip_writer_set_comment(archiver, comment.str().c_str());

‎src/apps/ociocheck/main.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ int main(int argc, const char **argv)
504504

505505
std::string cacheID;
506506
bool isArchivable = false;
507+
const bool minimal = false;
507508
try
508509
{
509510
LogGuard logGuard;
@@ -512,7 +513,7 @@ int main(int argc, const char **argv)
512513
std::cout << logGuard.output();
513514

514515
cacheID = config->getCacheID();
515-
isArchivable = config->isArchivable();
516+
isArchivable = config->isArchivable(minimal);
516517

517518
// Passed if there are no Error level logs.
518519
StringUtils::StringVec svec = StringUtils::SplitByLines(logGuard.output());

‎tests/cpu/OCIOZArchive_tests.cpp‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
116116

117117
std::istringstream iss;
118118
iss.str(CONFIG);
119+
const bool minimal = false;
119120

120121
OCIO::ConfigRcPtr cfg;
121122
OCIO_CHECK_NO_THROW(cfg = OCIO::Config::CreateFromStream(iss)->createEditableCopy());
@@ -138,74 +139,74 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
138139

139140
// Valid search path.
140141
cfg->setSearchPath("luts");
141-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
142+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
142143

143144
cfg->setSearchPath(R"(luts/myluts1)");
144-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
145+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
145146

146147
cfg->setSearchPath(R"(luts\myluts1)");
147-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
148+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
148149

149150
// Valid Search path starting with "./" or ".\".
150151
cfg->setSearchPath(R"(./myLuts)");
151-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
152+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
152153

153154
cfg->setSearchPath(R"(.\myLuts)");
154-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
155+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
155156

156157
// Valid search path starting with "./" or ".\" and a context variable.
157158
cfg->setSearchPath(R"(./$SHOT/myluts)");
158-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
159+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
159160

160161
cfg->setSearchPath(R"(.\$SHOT\myluts)");
161-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
162+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
162163

163164
cfg->setSearchPath(R"(luts/$SHOT)");
164-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
165+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
165166

166167
cfg->setSearchPath(R"(luts/$SHOT/luts1)");
167-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
168+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
168169

169170
cfg->setSearchPath(R"(luts\$SHOT)");
170-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
171+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
171172

172173
cfg->setSearchPath(R"(luts\$SHOT\luts1)");
173-
OCIO_CHECK_EQUAL(true, cfg->isArchivable());
174+
OCIO_CHECK_EQUAL(true, cfg->isArchivable(minimal));
174175

175176
/*
176177
* Illegal scenarios
177178
*/
178179

179180
// Illegal search path starting with "..".
180181
cfg->setSearchPath(R"(luts:../luts)");
181-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
182+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
182183

183184
cfg->setSearchPath(R"(luts:..\myLuts)");
184-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
185+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
185186

186187
// Illegal search path starting with a context variable.
187188
cfg->setSearchPath(R"(luts:$SHOT)");
188-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
189+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
189190

190191
// Illegal search path with absolute path.
191192
cfg->setSearchPath(R"(luts:/luts)");
192-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
193+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
193194

194195
cfg->setSearchPath(R"(luts:/$SHOT)");
195-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
196+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
196197

197198
#ifdef _WIN32
198199
cfg->clearSearchPaths();
199200
cfg->addSearchPath(R"(C:\luts)");
200-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
201+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
201202

202203
cfg->clearSearchPaths();
203204
cfg->addSearchPath(R"(C:\)");
204-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
205+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
205206

206207
cfg->clearSearchPaths();
207208
cfg->addSearchPath(R"(C:\$SHOT)");
208-
OCIO_CHECK_EQUAL(false, cfg->isArchivable());
209+
OCIO_CHECK_EQUAL(false, cfg->isArchivable(minimal));
209210
#endif
210211
}
211212

@@ -224,7 +225,7 @@ OCIO_ADD_TEST(OCIOZArchive, is_config_archivable)
224225
cs->setTransform(ft, OCIO::COLORSPACE_DIR_TO_REFERENCE);
225226
cfg->addColorSpace(cs);
226227

227-
OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable());
228+
OCIO_CHECK_EQUAL(isArchivable, cfg->isArchivable(minimal));
228229

229230
cfg->removeColorSpace("csTest");
230231
};

0 commit comments

Comments
 (0)