Commit d987b49a authored by Graydon Hoare's avatar Graydon Hoare

More hacking on the snapshot system.

parent ed40c85a
......@@ -108,7 +108,6 @@ GENERATED :=
%:: s.%
%:: SCCS/s.%
######################################################################
# Standard library variables
######################################################################
......@@ -135,6 +134,13 @@ SREQ1 := stage1/rustc$(X) $(LREQ) stage2/glue.o stage2/$(CFG_STDLIB)
SREQ2 := stage2/rustc$(X) $(LREQ) stage3/glue.o stage3/$(CFG_STDLIB)
######################################################################
# Exports for sub-utilities
######################################################################
export CFG_SRC_DIR
######################################################################
# Single-target rules
######################################################################
......@@ -179,5 +185,6 @@ include $(CFG_SRC_DIR)/mk/rustllvm.mk
include $(CFG_SRC_DIR)/mk/docs.mk
include $(CFG_SRC_DIR)/mk/tests.mk
include $(CFG_SRC_DIR)/mk/dist.mk
include $(CFG_SRC_DIR)/mk/snap.mk
include $(CFG_SRC_DIR)/mk/clean.mk
include $(CFG_SRC_DIR)/mk/autodep.mk
......@@ -9,34 +9,15 @@ def snap_filename_hash_part(snap):
raise Exception("unable to find hash in filename: " + snap)
return match.group(1)
def get_snapshot_and_check_hash(snap):
hsh = snap_filename_hash_part(snap)
h = hashlib.sha1()
url = download_url_base + "/" + snap
print "downloading " + url
u = urllib2.urlopen(url)
print "checking hash on download"
data = u.read()
h.update(data)
if h.hexdigest() != hsh:
raise Exception("hash check failed on " + snap)
print "hash ok"
with open(os.path.join(download_dir_base, snap), "w+b") as f:
f.write(data)
return True
def unpack_snapshot(snap):
dl_path = os.path.join(download_dir_base, snap)
print "opening snapshot " + dl_path
print("opening snapshot " + dl_path)
tar = tarfile.open(dl_path)
kernel = get_kernel()
for name in snapshot_files[kernel]:
p = os.path.join("rust-stage0", name)
fp = os.path.join("stage0", name)
print "extracting " + fp
print("extracting " + fp)
tar.extract(p, download_unpack_base)
tp = os.path.join(download_unpack_base, p)
shutil.move(tp, fp)
......@@ -80,16 +61,16 @@ def determine_last_snapshot_for_platform():
# Main
snap = determine_last_snapshot_for_platform()
print "determined most recent snapshot: " + snap
dl = os.path.join(download_dir_base, snap)
if (os.path.exists(dl)):
if (snap_filename_hash_part(snap) == hash_file(dl)):
print "found existing download with ok hash"
else:
print "bad hash on existing download, re-fetching"
get_snapshot_and_check_hash(snap)
url = download_url_base + "/" + snap
print("determined most recent snapshot: " + snap)
if (not os.path.exists(dl)):
get_url_to_file(url, dl)
if (snap_filename_hash_part(snap) == hash_file(dl)):
print("got download with ok hash")
else:
print "no cached download, fetching"
get_snapshot_and_check_hash(snap)
raise Exception("bad hash on download")
unpack_snapshot(snap)
......@@ -21,4 +21,4 @@ file1 = full_snapshot_name(date, rev, kernel, cpu, h)
shutil.move(file0, file1)
print file1
print(file1)
import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile
import re, os, sys, hashlib, tarfile, shutil, subprocess, tempfile
snapshotfile = "snapshots.txt"
src_dir = os.getenv("CFG_SRC_DIR")
if not src_dir:
raise Exception("missing env var CFG_SRC_DIR")
snapshotfile = os.path.join(src_dir, "snapshots.txt")
download_url_base = "http://dl.rust-lang.org/stage0-snapshots"
download_dir_base = "dl"
download_unpack_base = os.path.join(download_dir_base, "unpack")
......@@ -59,11 +63,16 @@ def get_cpu():
def get_platform():
return "%s-%s" % (get_kernel(), get_cpu())
def scrub(b):
if sys.version_info >= (3,) and type(b) == bytes:
return b.decode('ascii')
else:
return b
def cmd_out(cmdline):
p = subprocess.Popen(cmdline,
stdout=subprocess.PIPE)
return p.communicate()[0].strip()
return scrub(p.communicate()[0].strip())
def local_rev_info(field):
......@@ -82,8 +91,10 @@ def local_rev_short_sha():
def local_rev_committer_date():
return local_rev_info("ci")
def get_url_to_file(u,f):
subprocess.check_call(["curl", "-o", f, u])
def hash_file(x):
h = hashlib.sha1()
h.update(open(x).read())
return h.hexdigest()
h.update(open(x, "rb").read())
return scrub(h.hexdigest())
S 2011-05-02 ed40c85
winnt-i386 e69c11fbc62639ac3a3eef7ea36c9ad77209e2b1
S 2011-04-29 7b95b5c
linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment