Commit cb53065a authored by Graydon Hoare's avatar Graydon Hoare

Initial sketch of snapshot support code, based on code from marijn.

parent 2fc58fc6
#!/usr/bin/env python
import os, tarfile, hashlib, re, shutil
from snapshot import *
def snap_filename_hash_part(snap):
match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
if not match:
raise Exception("unable to find hash in filename: " + snap)
def get_snapshot_and_check_hash(snap):
hsh = snap_filename_hash_part(snap)
h = hashlib.sha1()
url = download_url_base + "/" + snap
print "downloading " + url
u = urllib2.urlopen(url)
print "checking hash on download"
data =
if h.hexdigest() != hsh:
raise Exception("hash check failed on " + snap)
print "hash ok"
with open(os.path.join(download_dir_base, snap), "w+b") as f:
return True
def unpack_snapshot(snap):
dl_path = os.path.join(download_dir_base, snap)
print "opening snapshot " + dl_path
tar =
kernel = get_kernel()
for name in snapshot_files[kernel]:
p = os.path.join("rust-stage0", name)
fp = os.path.join("stage0", name)
print "extracting " + fp
tar.extract(p, download_unpack_base)
tp = os.path.join(download_unpack_base, p)
shutil.move(tp, fp)
def determine_last_snapshot_for_platform():
lines = open(snapshotfile).readlines();
platform = get_platform()
found = False
hsh = None
date = None
rev = None
for ln in range(len(lines) - 1, -1, -1):
parsed = parse_line(ln, lines[ln])
if (not parsed): continue
if parsed["type"] == "file":
if parsed["platform"] == platform:
hsh = parsed["hash"]
elif parsed["type"] == "snapshot":
date = parsed["date"]
rev = parsed["rev"]
found = True
elif parsed["type"] == "transition" and not foundSnapshot:
raise Exception("working on a transition, not updating stage0")
if not found:
raise Exception("no snapshot entries in file")
if not hsh:
raise Exception("no snapshot file found for platform %s, rev %s" %
(platform, rev))
return full_snapshot_name(date, rev, get_kernel(), get_cpu(), hsh)
# Main
snap = determine_last_snapshot_for_platform()
print "determined most recent snapshot: " + snap
dl = os.path.join(download_dir_base, snap)
if (os.path.exists(dl)):
if (snap_filename_hash_part(snap) == hash_file(dl)):
print "found existing download with ok hash"
print "bad hash on existing download, re-fetching"
print "no cached download, fetching"
#!/usr/bin/env python
import shutil, tarfile
from snapshot import *
kernel = get_kernel()
cpu = get_cpu()
rev = local_rev_short_sha()
date = local_rev_committer_date().split()[0]
file0 = partial_snapshot_name(date, rev, kernel, cpu)
tar =, "w:bz2")
for name in snapshot_files[kernel]:
tar.add(os.path.join("stage2", name),
os.path.join("rust-stage0", name))
h = hash_file(file0)
file1 = full_snapshot_name(date, rev, kernel, cpu, h)
shutil.move(file0, file1)
print file1
import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile
snapshotfile = "snapshots.txt"
download_url_base = ""
download_dir_base = "dl"
download_unpack_base = os.path.join(download_dir_base, "unpack")
snapshot_files = {
"linux": ["rustc", "glue.o", "" ],
"macos": ["rustc", "glue.o", "libstd.dylib" ],
"winnt": ["rustc.exe", "glue.o", "std.dll" ]
def parse_line(n, line):
global snapshotfile
if re.match(r"\s*$", line): return None
match = re.match(r"\s+([\w_-]+) ([a-fA-F\d]{40})\s*$", line)
if match:
return { "type": "file",
"hash": }
match = re.match(r"([ST]) (\d{4}-\d{2}-\d{2}) ([a-fA-F\d]+)\s*$", line);
if (not match):
raise Exception("%s:%d:E syntax error" % (snapshotfile, n))
ttype = "snapshot"
if ( == "T"):
ttype = "transition"
return {"type": ttype,
def partial_snapshot_name(date, rev, kernel, cpu):
return ("rust-stage0-%s-%s-%s-%s.tar.bz2"
% (date, rev, kernel, cpu))
def full_snapshot_name(date, rev, kernel, cpu, hsh):
return ("rust-stage0-%s-%s-%s-%s-%s.tar.bz2"
% (date, rev, kernel, cpu, hsh))
def get_kernel():
if == "nt":
return "winnt"
kernel = os.uname()[0].lower()
if kernel == "darwin":
kernel = "macos"
return kernel
def get_cpu():
# return os.uname()[-1].lower()
return "i386"
def get_platform():
return "%s-%s" % (get_kernel(), get_cpu())
def cmd_out(cmdline):
p = subprocess.Popen(cmdline,
return p.communicate()[0].strip()
def local_rev_info(field):
return cmd_out(["git", "log", "-n", "1",
"--format=%%%s" % field, "HEAD"])
def local_rev_full_sha():
return local_rev_info("H").split()[0]
def local_rev_short_sha():
return local_rev_info("h").split()[0]
def local_rev_committer_date():
return local_rev_info("ci")
def hash_file(x):
h = hashlib.sha1()
return h.hexdigest()
S 2011-04-29 7b95b5c
linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00
winnt-i386 7d27adcc5e0c111e3221751962a7df0bcb9a9288
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment