hgmolib: make s3 bundle generation code Python 3 compatible (Bug 1541166) r=glob
authorConnor Sheehan <sheehan@mozilla.com>
Wed, 30 Oct 2019 00:51:39 +0000
changeset 7278 76731091d482bba04d538061647da7fdaf2d75fe
parent 7277 d3c8e2f2e9469f1298a5012fce3dbea4ef654e5b
child 7279 e73e374878f3e1a30ab7524bb345925e85442e00
push id3593
push usercosheehan@mozilla.com
push dateWed, 30 Oct 2019 14:30:52 +0000
treeherderversion-control-tools@76731091d482 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglob
bugs1541166
hgmolib: make s3 bundle generation code Python 3 compatible (Bug 1541166) r=glob Some simple changes to read objects into strings instead of bytes, and encode data from a subprocess. Differential Revision: https://phabricator.services.mozilla.com/D50345
hgserver/hgmolib/hgmolib/generate_hg_s3_bundles.py
--- a/hgserver/hgmolib/hgmolib/generate_hg_s3_bundles.py
+++ b/hgserver/hgmolib/hgmolib/generate_hg_s3_bundles.py
@@ -292,21 +292,23 @@ def generate_bundles(repo, upload=True, 
 
     hg_stat = os.stat(os.path.join(repo_full, '.hg'))
     uid = hg_stat.st_uid
     gid = hg_stat.st_gid
 
     # Bundle files are named after the tip revision in the repository at
     # the time the bundle was created. This is the easiest way to name
     # bundle files.
-    tip = subprocess.check_output([HG, '-R', repo_full, 'log', '-r', 'tip', '-T', '{node}'])
+    tip = subprocess.check_output(
+        [HG, '-R', repo_full, 'log', '-r', 'tip', '-T', '{node}']
+    ).decode('latin-1')
     print('tip is %s' % tip)
 
     with open(os.path.join(repo_full, '.hg', 'requires'), 'rb') as fh:
-        generaldelta = 'generaldelta\n' in fh.readlines()
+        generaldelta = b'generaldelta\n' in fh.readlines()
 
     if not generaldelta:
         raise Exception('non-generaldelta repo not supported: %s' % repo_full)
 
     bundle_path = os.path.join(BUNDLE_ROOT, repo)
 
     # Create directory to hold bundle files.
     try:
@@ -440,17 +442,17 @@ def generate_bundles(repo, upload=True, 
 
     backup_path = os.path.join(repo_full, '.hg', 'clonebundles.manifest.old')
     clonebundles_path = os.path.join(repo_full, '.hg', 'clonebundles.manifest')
 
     if os.path.exists(clonebundles_path):
         print('Copying %s -> %s' % (clonebundles_path, backup_path))
         shutil.copy2(clonebundles_path, backup_path)
 
-    with open(clonebundles_path, 'wb') as fh:
+    with open(clonebundles_path, 'w') as fh:
         fh.write('\n'.join(clonebundles_manifest))
 
     # Ensure manifest is owned by same user who owns repo and has sane
     # permissions.
     # TODO we can't do this yet since the "hg" user isn't a member of the
     # scm_* groups.
     # os.chown(clonebundles_path, uid, gid)
     os.chmod(clonebundles_path, 0o664)
@@ -488,17 +490,17 @@ def generate_index(repos):
 
         entries.append(HTML_ENTRY.format(**opts))
 
     html = HTML_INDEX % ('\n'.join(entries),
                          datetime.datetime.utcnow().isoformat())
 
     # We rely on the mtime of this file for monitoring to ensure
     # bundle generation is working.
-    with open(os.path.join(BUNDLE_ROOT, 'index.html'), 'wb') as fh:
+    with open(os.path.join(BUNDLE_ROOT, 'index.html'), 'w') as fh:
         fh.write(html)
 
     return html
 
 
 def upload_index(html):
     for host, bucket, region in S3_HOSTS:
         client = boto3.client('s3', region_name=region)
@@ -523,17 +525,17 @@ def generate_json_manifest(repos):
         d[repo] = {}
         for t, (path, size) in bundles.items():
             d[repo][t] = {
                 'path': path,
                 'size': size,
             }
 
     data = json.dumps(d, sort_keys=True, indent=4)
-    with open(os.path.join(BUNDLE_ROOT, 'bundles.json'), 'wb') as fh:
+    with open(os.path.join(BUNDLE_ROOT, 'bundles.json'), 'w') as fh:
         fh.write(data)
 
     return data
 
 
 def upload_json_manifest(data):
     for host, bucket, region in S3_HOSTS:
         c = boto3.client('s3', region_name=region)
@@ -551,17 +553,17 @@ def main():
     parser.add_argument('-f', help='file to read repository list from')
     parser.add_argument('--no-upload', action='store_true',
                         help='do not upload to servers (useful for testing)')
 
     repos = []
 
     args, remaining = parser.parse_known_args()
     if args.f:
-        with open(args.f, 'rb') as fh:
+        with open(args.f, 'r') as fh:
             items = [l.rstrip() for l in fh]
     else:
         items = remaining
 
     for item in items:
         attrs = {}
         fields = item.split()
         # fields[0] is the repo path.