Mercurial > oi > oi-build
changeset 34:d20b10eba317 build-149
prepare for zone based component builds
author | Norm Jacobs <Norm.Jacobs@Sun.COM> |
---|---|
date | Sun, 19 Sep 2010 15:50:53 -0500 |
parents | 371c8e56136d |
children | 9c84697348fb |
files | components/Makefile make-rules/prep.mk make-rules/shared-macros.mk tools/bass-o-matic tools/bass-o-matic.py tools/fetch.py tools/unpack.py tools/userland-fetch tools/userland-unpack |
diffstat | 9 files changed, 561 insertions(+), 525 deletions(-) [+] |
line wrap: on
line diff
--- a/components/Makefile Thu Sep 02 11:08:43 2010 -0500 +++ b/components/Makefile Sun Sep 19 15:50:53 2010 -0500 @@ -31,9 +31,9 @@ # components.mk is auto-generated by the build tools. It populates the # COMPONENT_DIRS macro with a list of all directories in the workspace that # contain a component. -components.mk: Makefile # $(WS_TOP)/components +components.mk: # Makefile # $(WS_TOP)/components @echo "Generating component list..." - @$(TOOLS)/bass-o-matic.py --workspace=$(WS_TOP) --components=paths \ + @$(BASS_O_MATIC) --workspace=$(WS_TOP) --components=paths \ | sed -e 's;^$(shell pwd)/;COMPONENT_DIRS += ;g' >$@ include components.mk @@ -47,21 +47,22 @@ validate: TARGET = validate clean: TARGET = clean clobber: TARGET = clobber -prep build install publish: LOG = >$(WS_LOGS)/$(TARGET):$@.log 2>&1 +prep build install publish: TEMPLATE_ZONE=$(ZONE) +prep build install publish: LOG = >$(WS_LOGS)/$@.$(TARGET).log 2>&1 .DEFAULT: publish -.PARALLEL: $(COMPONENT_DIRS) +download prep build install publish validate: $(COMPONENT_DIRS) -download prep build install publish validate: setup $(COMPONENT_DIRS) +COMPONENT_DIRS.nosetup = $(COMPONENT_DIRS:%=%.nosetup) -clean clobber: $(COMPONENT_DIRS) -ifeq ($(findstring file://, $(PKG_REPO)),file://) - $(RM) -r $(PKG_REPO:file://%=%) $(WS_LOGS) -endif - $(RM) components.mk depends.mk +clean: $(COMPONENT_DIRS:%=%.nosetup) + $(RM) components.mk depends.mk .profile -setup: $(WS_LOGS) repo tools +clobber: $(COMPONENT_DIRS:%=%.nosetup) clean + $(RM) -r $(WS_REPO) $(WS_LOGS) + +setup: $(WS_LOGS) repo tools .profile $(WS_LOGS): $(MKDIR) $@ @@ -73,13 +74,27 @@ tools: @cd ../tools ; echo "building tools..." ; $(GMAKE) setup +# $(WS_COMPONENTS) is the home directory for the zone user, so create a profile +# to pass a few things on to zone based builds +.profile: + echo "PATH=$(PATH)" >>$@ + echo "WS_TOP=$(WS_TOP)" >>$@ + echo "export PATH WS_TOP" >>$@ + $(COMPONENT_DIRS): setup FORCE - @cd $@ ; echo "$(TARGET) $@" ; $(GMAKE) $(TARGET) $(LOG) + @cd $@ ; echo "$(TARGET) $@" ; \ + $(BASS_O_MATIC) $(TEMPLATE_ZONE:%=--template-zone %) \ + $(@:%=--component %) --make $(TARGET) $(LOG) + +$(COMPONENT_DIRS.nosetup): FORCE + @cd $(@:%.nosetup=%) ; echo "$(TARGET) $(@:%.nosetup=%)" ; \ + $(BASS_O_MATIC) $(TEMPLATE_ZONE:%=--template-zone %) \ + $(@:%.nosetup=--component %) --make $(TARGET) $(LOG) # depends.mk is auto-generated by the build tools, bass-o-matic.py in particular depends.mk: components.mk @echo "Generating component dependencies..." - @$(TOOLS)/bass-o-matic.py --workspace=$(WS_TOP) --components=depend \ + @$(BASS_O_MATIC) --workspace=$(WS_TOP) --components=depend \ | sed -e 's;$(shell pwd)/;;g' >$@ include depends.mk
--- a/make-rules/prep.mk Thu Sep 02 11:08:43 2010 -0500 +++ b/make-rules/prep.mk Sun Sep 19 15:50:53 2010 -0500 @@ -21,8 +21,8 @@ # Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. # -UNPACK = $(TOOLS)/unpack.py -FETCH = $(TOOLS)/fetch.py +UNPACK = $(WS_TOOLS)/userland-unpack +FETCH = $(WS_TOOLS)/userland-fetch ARCHIVES += $(COMPONENT_ARCHIVE) CLEAN_PATHS += $(COMPONENT_SRC)
--- a/make-rules/shared-macros.mk Thu Sep 02 11:08:43 2010 -0500 +++ b/make-rules/shared-macros.mk Sun Sep 19 15:50:53 2010 -0500 @@ -40,18 +40,22 @@ PYTHON_VERSION = 2.6 PYTHON_VERSIONS = 2.4 2.6 -TOOLS = $(WS_TOP)/tools WS_LOGS = $(WS_TOP)/logs -MAKE_RULES = $(WS_TOP)/make-rules +WS_REPO = $(WS_TOP)/repo +WS_TOOLS = $(WS_TOP)/tools +WS_MAKE_RULES = $(WS_TOP)/make-rules +WS_COMPONENTS = $(WS_TOP)/components -PKG_REPO = file://$(WS_TOP)/repo +BASS_O_MATIC = $(WS_TOOLS)/bass-o-matic + +PKG_REPO = file:$(WS_REPO) PROTO_DIR = $(shell pwd)/$(COMPONENT_SRC)/installed-prototype-$(MACH) # work around _TIME, _DATE, embedded date chatter in component builds # to use, set TIME_CONSTANT in the component Makefile and add $(CONSTANT_TIME) # to the appropriate {CONFIGURE|BUILD|INSTALL}_ENV -CONSTANT_TIME = LD_PRELOAD_32=$(TOOLS)/time-$(MACH32).so -CONSTANT_TIME += LD_PRELOAD_64=$(TOOLS)/time-$(MACH64).so +CONSTANT_TIME = LD_PRELOAD_32=$(WS_TOOLS)/time-$(MACH32).so +CONSTANT_TIME += LD_PRELOAD_64=$(WS_TOOLS)/time-$(MACH64).so CONSTANT_TIME += TIME_CONSTANT=$(TIME_CONSTANT) # set MACH from uname -p to either sparc or i386
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/bass-o-matic Sun Sep 19 15:50:53 2010 -0500 @@ -0,0 +1,186 @@ +#!/usr/bin/python2.6 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. +# +# +# bass-o-matic.py +# A simple program to enumerate components in the userland gate and report +# on dependency related information. +# + +import os +import sys +import re +import subprocess + +# Locate SCM directories containing Userland components by searching from +# from a supplied top of tree for .p5m files. Once a .p5m file is located, +# that directory is added to the list and no children are searched. +def FindComponentPaths(path, debug=None): + expression = re.compile(".+\.p5m$", re.IGNORECASE) + + paths = [] + + if debug: + print >>debug, "searching %s for component directories" % path + + for dirpath, dirnames, filenames in os.walk(path + '/components'): + found = 0 + + for name in filenames: + if expression.match(name): + if debug: + print >>debug, "found %s" % dirpath + paths.append(dirpath) + del dirnames[:] + break + + return sorted(paths) + +class BassComponent: + def __init__(self, path=None, debug=None): + self.debug = debug + self.path = path + if path: + # get supplied packages (cd path ; gmake print-package-names) + self.supplied_packages = self.run_make(path, 'print-package-names') + + # get supplied paths (cd path ; gmake print-package-paths) + self.supplied_paths = self.run_make(path, 'print-package-paths') + + # get required paths (cd path ; gmake print-required-paths) + self.required_paths = self.run_make(path, 'print-required-paths') + + def required(self, component): + result = False + + s1 = set(self.required_paths) + s2 = set(component.supplied_paths) + if s1.intersection(s2): + result = True + + return result + + def run_make(self, path, targets): + + result = list() + + if self.debug: + print >>self.debug, "Executing 'gmake %s' in %s" % (targets, path) + + proc = subprocess.Popen(['gmake', targets], cwd=path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = proc.stdout + + for out in p: + result.append(out) + + if self.debug: + proc.wait() + if proc.returncode != 0: + print >>self.debug, "exit: %d, %s" % (proc.returncode, proc.stderr.read()) + + return result + + def __str__(self): + result = "Component:\n\tPath: %s\n" % self.path + result = result + "\tProvides Package(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_packages) + result = result + "\tProvides Path(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_paths) + result = result + "\tRequired Path(s):\n\t\t%s\n" % '\t\t'.join(self.required_paths) + + return result + +def usage(): + print "Usage: %s [-c|--components=(path|depend)] [-z|--zone (zone)]" % (sys.argv[0].split('/')[-1]) + sys.exit(1) + +def main(): + import getopt + import sys + + # FLUSH STDOUT + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + + components = {} + debug=None + components_arg=None + make_arg=None + component_arg=None + template_zone=None + workspace = os.getenv('WS_TOP') + + try: + opts, args = getopt.getopt(sys.argv[1:], "w:c:d", + [ "debug", "workspace=", "components=", + "make", "component=", "template-zone=" ]) + except getopt.GetoptError, err: + print str(err) + usage() + + for opt, arg in opts: + if opt in [ "-w", "--workspace" ]: + workspace = arg + elif opt in [ "-l", "--components" ]: + components_arg = arg + elif opt in [ "--make" ]: + make_arg = True + elif opt in [ "--component" ]: + component_arg = arg + elif opt in [ "--template-zone" ]: + template_zone = arg + elif opt in [ "-d", "--debug" ]: + debug = sys.stdout + else: + assert False, "unknown option" + + component_paths = FindComponentPaths(workspace, debug) + + if make_arg: + if template_zone: + print "using template zone %s to create a build environment for %s to run '%s'" % (template_zone, component_arg, ['gmake'] + args) + proc = subprocess.Popen(['gmake'] + args) + proc.wait() + sys.exit(0) + + if components_arg: + if components_arg in [ 'path', 'paths', 'dir', 'dirs', 'directories' ]: + for path in component_paths: + print "%s" % path + + elif components_arg in [ 'depend', 'dependencies' ]: + for path in component_paths: + components[path] = BassComponent(path, debug) + + for c_path in components.keys(): + component = components[c_path] + + for d_path in components.keys(): + if (c_path != d_path and + component.required(components[d_path])): + print "%s: %s" % (c_path, d_path) + + sys.exit(0) + + sys.exit(1) + +if __name__ == "__main__": + main()
--- a/tools/bass-o-matic.py Thu Sep 02 11:08:43 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,169 +0,0 @@ -#!/usr/bin/python2.6 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. -# -# -# bass-o-matic.py -# A simple program to enumerate components in the userland gate and report -# on dependency related information. -# - -import os -import sys -import re - -# Locate SCM directories containing Userland components by searching from -# from a supplied top of tree for .p5m files. Once a .p5m file is located, -# that directory is added to the list and no children are searched. -def FindComponentPaths(path, debug=None): - expression = re.compile(".+\.p5m$", re.IGNORECASE) - - paths = [] - - if debug: - print >>debug, "searching %s for component directories" % path - - for dirpath, dirnames, filenames in os.walk(path + '/components'): - found = 0 - - for name in filenames: - if expression.match(name): - if debug: - print >>debug, "found %s" % dirpath - paths.append(dirpath) - del dirnames[:] - break - - return sorted(paths) - -class BassComponent: - def __init__(self, path=None, debug=None): - self.debug = debug - self.path = path - if path: - # get supplied packages (cd path ; gmake print-package-names) - self.supplied_packages = self.run_make(path, 'print-package-names') - - # get supplied paths (cd path ; gmake print-package-paths) - self.supplied_paths = self.run_make(path, 'print-package-paths') - - # get required paths (cd path ; gmake print-required-paths) - self.required_paths = self.run_make(path, 'print-required-paths') - - def required(self, component): - result = False - - s1 = set(self.required_paths) - s2 = set(component.supplied_paths) - if s1.intersection(s2): - result = True - - return result - - def run_make(self, path, targets): - import subprocess - - result = list() - - if self.debug: - print >>self.debug, "Executing 'gmake %s' in %s" % (targets, path) - - proc = subprocess.Popen(['gmake', targets], cwd=path, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p = proc.stdout - - for out in p: - result.append(out) - - if self.debug: - proc.wait() - if proc.returncode != 0: - print >>self.debug, "exit: %d, %s" % (proc.returncode, proc.stderr.read()) - - return result - - def __str__(self): - result = "Component:\n\tPath: %s\n" % self.path - result = result + "\tProvides Package(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_packages) - result = result + "\tProvides Path(s):\n\t\t%s\n" % '\t\t'.join(self.supplied_paths) - result = result + "\tRequired Path(s):\n\t\t%s\n" % '\t\t'.join(self.required_paths) - - return result - -def usage(): - print "Usage: %s [-c|--components=(path|depend)] [-z|--zone (zone)]" % (sys.argv[0].split('/')[-1]) - sys.exit(1) - -def main(): - import getopt - import sys - - # FLUSH STDOUT - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - - components = {} - debug=None - components_arg=None - workspace = os.getenv('WS_TOP') - - try: - opts, args = getopt.getopt(sys.argv[1:], "w:c:d", - [ "debug", "workspace=", "components=" ]) - except getopt.GetoptError, err: - print str(err) - usage() - - for opt, arg in opts: - if opt in [ "-w", "--workspace" ]: - workspace = arg - elif opt in [ "-l", "--components" ]: - components_arg = arg - elif opt in [ "-d", "--debug" ]: - debug = sys.stdout - else: - assert False, "unknown option" - - component_paths = FindComponentPaths(workspace, debug) - - if components_arg: - if components_arg in [ 'path', 'paths', 'dir', 'dirs', 'directories' ]: - for path in component_paths: - print "%s" % path - - elif components_arg in [ 'depend', 'dependencies' ]: - for path in component_paths: - components[path] = BassComponent(path, debug) - - for c_path in components.keys(): - component = components[c_path] - - for d_path in components.keys(): - if (c_path != d_path and - component.required(components[d_path])): - print "%s: %s" % (c_path, d_path) - - sys.exit(0) - - sys.exit(1) - -if __name__ == "__main__": - main()
--- a/tools/fetch.py Thu Sep 02 11:08:43 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,192 +0,0 @@ -#!/usr/bin/python2.6 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. -# -# -# fetch.py - a file download utility -# -# A simple program similiar to wget(1), but handles local file copy, ignores -# directories, and verifies file hashes. -# - -import os -import sys - -def validate(filename, hash): - import hashlib - - try: - file = open(filename, 'r') - except IOError: - return False - - if (hash == None): - return True - - algorithm, value = hash.split(':') - try: - m = hashlib.new(algorithm) - except ValueError: - return False - - while True: - block = file.read() - m.update(block) - if block == '': - break - - return m.hexdigest() == value - -def download(url, filename = None, hash = None, verbose = False): - from urllib import splittype, urlopen - import hashlib - - src = None - filename = None - - #print "Downloading %s from %s" % (filename, url) - - # open the download source - if splittype(url)[0]: - try: - src = urlopen(url) - except IOError: - return None - remote_name = src.geturl().split('/')[-1] - elif os.path.isfile(url): - os.symlink(url, filename) - return filename, filehash(filename) - src = open(url, 'r') - remote_name = url.split('/')[-1] - else: - return None - - if filename == None: - filename = remote_name - - # if we have a source to download, open a destination and copy the data - if src: - dst = open(filename, 'wb'); - if verbose: - print "Downloading %s from %s" % (filename, url) - while True: - block = src.read() - if block == '': - break; - dst.write(block) - dst.close() - - # return the name of the file that we downloaded the data to. - return filename - -def download_paths(search, filename): - tmp = os.getenv('DOWNLOAD_SEARCH_PATH') - if tmp: - search += tmp.split(' ') - base = os.path.basename(filename) - - urls = list() - - for path in search: - urls.append(path+'/'+base) - - return urls - -def usage(): - print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1]) - sys.exit(1) - -def main(): - from urllib import splittype, urlopen - import getopt - import sys - - # FLUSH STDOUT - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - - verbose = False - filename = None - url = None - hash = None - search = list() - - try: - opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v", - ["file=", "hash=", "search=", "url=", "verbose"]) - except getopt.GetoptError, err: - print str(err) - usage() - - for opt, arg in opts: - if opt in [ "-f", "--file" ]: - filename = arg - elif opt in [ "-h", "--hash" ]: - hash = arg - elif opt in [ "-s", "--search" ]: - search.append(arg) - elif opt in [ "-u", "--url" ]: - url = arg - elif opt in [ "-v", "--verbose" ]: - verbose = True - else: - assert False, "unknown option" - - if url == None: - usage() - - print "Fetching %s" % filename - - if validate(filename, hash): - print "\tcurrent copy is ok, nothing to do" - sys.exit(0) - - # generate a list of URLS to search for a suitable download - urls = download_paths(search, filename) - urls.append(url) - - for url in urls: - print "\tTrying %s..." % url, - - if os.path.isfile(url): - os.symlink(url, filename) - elif splittype(url)[0]: - if download(url, filename) == None: - print "failed" - continue - - print "retrieved...", - - if validate(filename, hash): - print "validated" - sys.exit(0) - else: - print "corrupt" - - try: - os.remove(filename) - except OSError: - pass - - sys.exit(1) - -if __name__ == "__main__": - main()
--- a/tools/unpack.py Thu Sep 02 11:08:43 2010 -0500 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,144 +0,0 @@ -#!/usr/bin/python2.6 -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE -# or http://www.opensolaris.org/os/licensing. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at usr/src/OPENSOLARIS.LICENSE. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -# -# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. -# -# -# unpack.py - an archive unpack utility -# -# A simple program to uncompress and unpack source archive files into a target -# directory and fix permissions if requested. -# - -import os -import sys - -def uncompress_unpack_commands(filename, verbose=False): - import re - - uncompress = "/bin/cat" - - if (re.search("(\.bz2|\.tbz|\.tbz2)$", filename) != None): - uncompress = "/usr/bin/bzip2 -dc" - elif (re.search("(\.gz|\.tgz)$", filename) != None): - uncompress = "/usr/bin/gzip -dc" - elif (re.search("(\.Z)$", filename) != None): - uncompress = "/usr/bin/uncompress -c" - elif (re.search("(\.7z)$", filename) != None): - uncompress = "/usr/bin/7z --s" - elif (re.search("(\.zip)$", filename) != None): - uncompress = "/usr/bin/unzip -qo" - - unpack = " | gtar -xf -" - - if (re.search("(\.zip)$", filename) != None): - unpack = "" - elif (re.search("(\.jar)$", filename) != None): - unpack = " | jar xf -" - - if (verbose == True): - print "command: %s %s %s" % (uncompress, filename, unpack) - - return uncompress, unpack - -# -# recurse down a directory tree opening permissions so that others may access -# files in the tree. -# -def fixup_permissions(dir, verbose): - for entry in os.listdir(dir): - import stat - - path = "%s/%s" % (dir, entry) - - st = os.lstat(path) - mode = stat.S_IMODE(st.st_mode) - mode |= (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - if stat.S_ISDIR(st.st_mode): - mode |= (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - if (stat.S_IMODE(st.st_mode) != mode): - if (verbose == True): - print "Changing %s from %4.4o to %4.4o" % (path, - stat.S_IMODE(st.st_mode), mode) - os.chmod(path, mode) - - if stat.S_ISDIR(st.st_mode): - fixup_permissions(path, verbose) - - -def usage(): - print "Usage: %s [-v|--verbose] [-f|--fix-permissions] [-r|--relocate-to (dir)] (file)" % (sys.argv[0].split('/')[-1]) - sys.exit(1) - -def main(): - import getopt - import sys - import tempfile - - verbose = False - permissions = None - relocate_to = None - - try: - opts, args = getopt.getopt(sys.argv[1:], "fr:v", - ["fix-permissions", "relocate-to=", "verbose"]) - except getopt.GetoptError, err: - print str(err) - usage() - - for opt, arg in opts: - if opt in [ "-v", "--verbose" ]: - verbose = True - elif opt in [ "-f", "--fix-permissions" ]: - permissions = True - elif opt in [ "-r", "--relocate-to" ]: - relocate_to = arg - else: - assert False, "unknown option" - - filename = ((args[0] == '/') and "%s" or "../%s") % args[0] - uncompress, unpack = uncompress_unpack_commands(filename) - tempdir = tempfile.mkdtemp(dir='.') - - # extract the archive contents - if (verbose == True): - print "cd %s ; %s %s%s" % (tempdir, uncompress, filename, - unpack) - os.system("cd %s ; %s %s%s" % (tempdir, uncompress, filename, unpack)) - - # open up the permissions on what we extracted - if permissions: - fixup_permissions(tempdir, verbose) - - if (relocate_to == None): - # move everything in the tempdir here - for entry in os.listdir(tempdir): - path= "%s/%s" % (tempdir, entry) - os.renames(path, entry) - else: - # rename the tempdir and open it's permissions - os.renames(tempdir, relocate_to) - os.chmod(relocate_to, 0755) - - -if __name__ == "__main__": - main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/userland-fetch Sun Sep 19 15:50:53 2010 -0500 @@ -0,0 +1,192 @@ +#!/usr/bin/python2.6 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. +# +# +# fetch.py - a file download utility +# +# A simple program similiar to wget(1), but handles local file copy, ignores +# directories, and verifies file hashes. +# + +import os +import sys + +def validate(filename, hash): + import hashlib + + try: + file = open(filename, 'r') + except IOError: + return False + + if (hash == None): + return True + + algorithm, value = hash.split(':') + try: + m = hashlib.new(algorithm) + except ValueError: + return False + + while True: + block = file.read() + m.update(block) + if block == '': + break + + return m.hexdigest() == value + +def download(url, filename = None, hash = None, verbose = False): + from urllib import splittype, urlopen + import hashlib + + src = None + filename = None + + #print "Downloading %s from %s" % (filename, url) + + # open the download source + if splittype(url)[0]: + try: + src = urlopen(url) + except IOError: + return None + remote_name = src.geturl().split('/')[-1] + elif os.path.isfile(url): + os.symlink(url, filename) + return filename, filehash(filename) + src = open(url, 'r') + remote_name = url.split('/')[-1] + else: + return None + + if filename == None: + filename = remote_name + + # if we have a source to download, open a destination and copy the data + if src: + dst = open(filename, 'wb'); + if verbose: + print "Downloading %s from %s" % (filename, url) + while True: + block = src.read() + if block == '': + break; + dst.write(block) + dst.close() + + # return the name of the file that we downloaded the data to. + return filename + +def download_paths(search, filename): + tmp = os.getenv('DOWNLOAD_SEARCH_PATH') + if tmp: + search += tmp.split(' ') + base = os.path.basename(filename) + + urls = list() + + for path in search: + urls.append(path+'/'+base) + + return urls + +def usage(): + print "Usage: %s [-v|--verbose] [-f|--file (file)] [-h|--hash (hash)] --url (url)" % (sys.argv[0].split('/')[-1]) + sys.exit(1) + +def main(): + from urllib import splittype, urlopen + import getopt + import sys + + # FLUSH STDOUT + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + + verbose = False + filename = None + url = None + hash = None + search = list() + + try: + opts, args = getopt.getopt(sys.argv[1:], "f:h:s:u:v", + ["file=", "hash=", "search=", "url=", "verbose"]) + except getopt.GetoptError, err: + print str(err) + usage() + + for opt, arg in opts: + if opt in [ "-f", "--file" ]: + filename = arg + elif opt in [ "-h", "--hash" ]: + hash = arg + elif opt in [ "-s", "--search" ]: + search.append(arg) + elif opt in [ "-u", "--url" ]: + url = arg + elif opt in [ "-v", "--verbose" ]: + verbose = True + else: + assert False, "unknown option" + + if url == None: + usage() + + print "Fetching %s" % filename + + if validate(filename, hash): + print "\tcurrent copy is ok, nothing to do" + sys.exit(0) + + # generate a list of URLS to search for a suitable download + urls = download_paths(search, filename) + urls.append(url) + + for url in urls: + print "\tTrying %s..." % url, + + if os.path.isfile(url): + os.symlink(url, filename) + elif splittype(url)[0]: + if download(url, filename) == None: + print "failed" + continue + + print "retrieved...", + + if validate(filename, hash): + print "validated" + sys.exit(0) + else: + print "corrupt" + + try: + os.remove(filename) + except OSError: + pass + + sys.exit(1) + +if __name__ == "__main__": + main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tools/userland-unpack Sun Sep 19 15:50:53 2010 -0500 @@ -0,0 +1,144 @@ +#!/usr/bin/python2.6 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or http://www.opensolaris.org/os/licensing. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. +# +# +# unpack.py - an archive unpack utility +# +# A simple program to uncompress and unpack source archive files into a target +# directory and fix permissions if requested. +# + +import os +import sys + +def uncompress_unpack_commands(filename, verbose=False): + import re + + uncompress = "/bin/cat" + + if (re.search("(\.bz2|\.tbz|\.tbz2)$", filename) != None): + uncompress = "/usr/bin/bzip2 -dc" + elif (re.search("(\.gz|\.tgz)$", filename) != None): + uncompress = "/usr/bin/gzip -dc" + elif (re.search("(\.Z)$", filename) != None): + uncompress = "/usr/bin/uncompress -c" + elif (re.search("(\.7z)$", filename) != None): + uncompress = "/usr/bin/7z --s" + elif (re.search("(\.zip)$", filename) != None): + uncompress = "/usr/bin/unzip -qo" + + unpack = " | gtar -xf -" + + if (re.search("(\.zip)$", filename) != None): + unpack = "" + elif (re.search("(\.jar)$", filename) != None): + unpack = " | jar xf -" + + if (verbose == True): + print "command: %s %s %s" % (uncompress, filename, unpack) + + return uncompress, unpack + +# +# recurse down a directory tree opening permissions so that others may access +# files in the tree. +# +def fixup_permissions(dir, verbose): + for entry in os.listdir(dir): + import stat + + path = "%s/%s" % (dir, entry) + + st = os.lstat(path) + mode = stat.S_IMODE(st.st_mode) + mode |= (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + if stat.S_ISDIR(st.st_mode): + mode |= (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + if (stat.S_IMODE(st.st_mode) != mode): + if (verbose == True): + print "Changing %s from %4.4o to %4.4o" % (path, + stat.S_IMODE(st.st_mode), mode) + os.chmod(path, mode) + + if stat.S_ISDIR(st.st_mode): + fixup_permissions(path, verbose) + + +def usage(): + print "Usage: %s [-v|--verbose] [-f|--fix-permissions] [-r|--relocate-to (dir)] (file)" % (sys.argv[0].split('/')[-1]) + sys.exit(1) + +def main(): + import getopt + import sys + import tempfile + + verbose = False + permissions = None + relocate_to = None + + try: + opts, args = getopt.getopt(sys.argv[1:], "fr:v", + ["fix-permissions", "relocate-to=", "verbose"]) + except getopt.GetoptError, err: + print str(err) + usage() + + for opt, arg in opts: + if opt in [ "-v", "--verbose" ]: + verbose = True + elif opt in [ "-f", "--fix-permissions" ]: + permissions = True + elif opt in [ "-r", "--relocate-to" ]: + relocate_to = arg + else: + assert False, "unknown option" + + filename = ((args[0] == '/') and "%s" or "../%s") % args[0] + uncompress, unpack = uncompress_unpack_commands(filename) + tempdir = tempfile.mkdtemp(dir='.') + + # extract the archive contents + if (verbose == True): + print "cd %s ; %s %s%s" % (tempdir, uncompress, filename, + unpack) + os.system("cd %s ; %s %s%s" % (tempdir, uncompress, filename, unpack)) + + # open up the permissions on what we extracted + if permissions: + fixup_permissions(tempdir, verbose) + + if (relocate_to == None): + # move everything in the tempdir here + for entry in os.listdir(tempdir): + path= "%s/%s" % (tempdir, entry) + os.renames(path, entry) + else: + # rename the tempdir and open it's permissions + os.renames(tempdir, relocate_to) + os.chmod(relocate_to, 0755) + + +if __name__ == "__main__": + main()