diff --git a/code/repo_sync b/code/repo_sync index 6a3d30e..16fcd32 100755 --- a/code/repo_sync +++ b/code/repo_sync @@ -519,7 +519,6 @@ class ReplicationError(Exception): '''A custom error when replication fails''' pass - def replicateURLtoFilesystem(full_url, root_dir=None, base_url=None, copy_only_if_missing=False, appendToFilename=''): @@ -542,6 +541,7 @@ def replicateURLtoFilesystem(full_url, root_dir=None, local_file_path = os.path.join(root_dir, relative_url) + appendToFilename local_dir_path = os.path.dirname(local_file_path) if copy_only_if_missing and os.path.exists(local_file_path): + reposadocommon.createCompressedFileCopy(local_file_path, True) return local_file_path if not os.path.exists(local_dir_path): try: @@ -552,6 +552,7 @@ def replicateURLtoFilesystem(full_url, root_dir=None, getURL(full_url, local_file_path) except CurlDownloadError, err: raise ReplicationError(err) + reposadocommon.createCompressedFileCopy(local_file_path) return local_file_path diff --git a/code/reposadolib/reposadocommon.py b/code/reposadolib/reposadocommon.py index 4d2ec90..c200bbe 100644 --- a/code/reposadolib/reposadocommon.py +++ b/code/reposadolib/reposadocommon.py @@ -46,6 +46,7 @@ import time import urlparse import warnings +import gzip from xml.parsers.expat import ExpatError from xml.dom import minidom @@ -275,6 +276,24 @@ def getDataFromPlist(filename): return {} +def createCompressedFileCopy(local_file_path, copy_only_if_missing=False): + '''Creates a gzipped copy of the given file at the same location with .gz suffix''' + local_gz_file_path = local_file_path + '.gz' + if not (copy_only_if_missing and os.path.exists(local_gz_file_path)): + try: + f_in = open(local_file_path, 'rb') + except (OSError, IOError), err: + print_stderr('Error: could not open file at %s: %s' % (local_file_path, err)) + try: + f_out = gzip.open(local_gz_file_path, 'wb') + f_out.writelines(f_in) + f_out.close() + except (OSError, IOError), err: + print_stderr('Error: could not create compressed file at %s: %s' % (local_gz_file_path, err)) + finally: + f_in.close() + + def getDownloadStatus(): '''Reads download status info from disk''' return getDataFromPlist('DownloadStatus.plist') @@ -447,7 +466,7 @@ def writeBranchCatalogs(localcatalogpath): product_key, branch, localcatalogname) plistlib.writePlist(catalog, branchcatalogpath) - + createCompressedFileCopy(branchcatalogpath) def writeAllLocalCatalogs(): '''Writes out all local and branch catalogs. Used when we purge products.''' @@ -489,6 +508,7 @@ def writeLocalCatalogs(applecatalogpath): # write raw (unstable/development) catalog # with all downloaded Apple updates enabled plistlib.writePlist(catalog, localcatalogpath) + createCompressedFileCopy(localcatalogpath) # now write filtered catalogs (branches) based on this catalog writeBranchCatalogs(localcatalogpath) diff --git a/docs/URL_rewrites.md b/docs/URL_rewrites.md index fa98b5e..3d504c6 100644 --- a/docs/URL_rewrites.md +++ b/docs/URL_rewrites.md @@ -17,26 +17,32 @@ If you are using Apache2 as your webserver, you may be able to configure mod_rew Here is an example .htaccess file you could place at the root of your Reposado repo: - RewriteEngine On - Options FollowSymLinks - RewriteBase / - RewriteCond %{HTTP_USER_AGENT} Darwin/8 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/index$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/9 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-leopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/10 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-leopard-snowleopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/11 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-lion-snowleopard-leopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/12 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/13 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/14 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog [L] - RewriteCond %{HTTP_USER_AGENT} Darwin/15 - RewriteRule ^index(.*)\.sucatalog$ content/catalogs/others/index-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog [L] - + AddEncoding x-gzip .gz + AddType text/plain .gz + Options FollowSymLinks + RewriteEngine On + RewriteBase / + RewriteCond %{HTTP_USER_AGENT} Darwin/16 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-10.12-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=8] + RewriteCond %{HTTP_USER_AGENT} Darwin/15 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-10.11-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=7] + RewriteCond %{HTTP_USER_AGENT} Darwin/14 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-10.10-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=6] + RewriteCond %{HTTP_USER_AGENT} Darwin/13 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-10.9-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=5] + RewriteCond %{HTTP_USER_AGENT} Darwin/12 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-mountainlion-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=4] + RewriteCond %{HTTP_USER_AGENT} Darwin/11 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-lion-snowleopard-leopard.merged-1$1.sucatalog$2 [S=3] + RewriteCond %{HTTP_USER_AGENT} Darwin/10 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-leopard-snowleopard.merged-1$1.sucatalog$2 [S=2] + RewriteCond %{HTTP_USER_AGENT} Darwin/9 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/others/index-leopard.merged-1$1.sucatalog$2 [S=1] + RewriteCond %{HTTP_USER_AGENT} Darwin/8 + RewriteRule ^/index(.*)\.sucatalog(\.gz)?$ /content/catalogs/index$1.sucatalog + RewriteCond %{HTTP:Accept-Encoding} gzip + RewriteRule ^(.+\.(sucatalog|dist))$ $1.gz + This requires Apache2 to be configured to actually pay attention to mod_rewrite rules in .htaccess files. See your Apache and mod_rewrite documentation for details.