Mercurial > illumos > illumos-gate
changeset 12899:5abbd90da85a
6593083 wsdiff should exclude PASS fields from lintdump output before comparison
6510912 wsdiff should make use of more than 1 CPU
6849536 provide option to wsdiff to produce sorted lists of files
6862904 wsdiff contains number of unhandled exceptions
6970144 activation files comparison no longer needed
| author | Vladimir Kotal <Vladimir.Kotal@Sun.COM> |
|---|---|
| date | Thu, 22 Jul 2010 10:19:00 +0200 |
| parents | 2e278bf762e0 |
| children | 8c8b525b1ad6 |
| files | usr/src/tools/scripts/nightly.sh usr/src/tools/scripts/wsdiff.1 usr/src/tools/scripts/wsdiff.py |
| diffstat | 3 files changed, 433 insertions(+), 137 deletions(-) [+] |
line wrap: on
line diff
--- a/usr/src/tools/scripts/nightly.sh Thu Jul 22 10:15:06 2010 +0800 +++ b/usr/src/tools/scripts/nightly.sh Thu Jul 22 10:19:00 2010 +0200 @@ -1138,14 +1138,15 @@ oldproto=$2 newproto=$3 - echo "\n==== Objects that differ since last build ($label) ====\n" | \ - tee -a $LOGFILE >> $mail_msg_file - wsdiff="wsdiff" [ "$t_FLAG" = y ] && wsdiff="wsdiff -t" - $wsdiff -r ${TMPDIR}/wsdiff.results $oldproto $newproto 2>&1 | \ + echo "\n==== Getting object changes since last build at `date`" \ + "($label) ====\n" | tee -a $LOGFILE >> $mail_msg_file + $wsdiff -s -r ${TMPDIR}/wsdiff.results $oldproto $newproto 2>&1 | \ tee -a $LOGFILE >> $mail_msg_file + echo "\n==== Object changes determined at `date` ($label) ====\n" | \ + tee -a $LOGFILE >> $mail_msg_file } #
--- a/usr/src/tools/scripts/wsdiff.1 Thu Jul 22 10:15:06 2010 +0800 +++ b/usr/src/tools/scripts/wsdiff.1 Thu Jul 22 10:19:00 2010 +0200 @@ -1,4 +1,3 @@ -.\" ident "%Z%%M% %I% %E% SMI" .\" " CDDL HEADER START .\" " .\" " The contents of this file are subject to the terms of the @@ -18,13 +17,13 @@ .\" " .\" " CDDL HEADER END .\" " -.\" "Copyright 2006 Sun Microsystems, Inc. All rights reserved. -.\" "Use is subject to license terms." -.TH wsdiff 1 "07 Jul 2006" +.\" " Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. +.\" " +.TH wsdiff 1 "15 Jul 2010" .I wsdiff \- report differences between proto area objects .SH SYNOPSIS -\fBwsdiff [-vVt] [-r \fIresults\fP] [-i \fIfilelist\fP] \fIold\fP \fInew\fP +\fBwsdiff [-dvVst] [-r \fIresults\fP] [-i \fIfilelist\fP] \fIold\fP \fInew\fP .LP .SH DESCRIPTION .IX "OS-Net build tools" "wsdiff" "" "\fBwsdiff\fP" @@ -41,6 +40,9 @@ with the source change. .SH OPTIONS .TP 10 +.B -d +Print debug information. The debug lines are prefixed with ##. +.TP 10 .B -v Do not truncate the diffs logged to the results file. By default wsdiff will truncate the length of a sufficiently long set of object diffs to preserve @@ -54,6 +56,11 @@ just the first. Because of the extra work involved, this may slow wsdiff down considerably. .TP 10 +.B -s +Produce sorted lists. This is handy when comparing multiple wsdiff outputs +because wsdiff runs in multithreaded mode so the list of differences is not +sorted and can differ between multiple runs. +.TP 10 .B -t Look for the onbld tools in $SRC/tools rather than /opt/onbld/bin .TP 10 @@ -170,6 +177,20 @@ invoke wsdiff at the end of the build. The list of changed objects will be reported in the nightly mail message, and a results file "wsdiff_results" will appear in the nightly log area. +.PP +\fBExample 5: Influencing the level of paralelism\fR +.PP +wsdiff spawns a number of threads by default after it determines the list +of files for comparison. Default number of threads is based on the number of +on-line CPUs present in the system. To set the number of threads for processing +to some other value the DMAKE_MAX_JOBS environment variable can be used: +.LP +.nf +$ DMAKE_MAX_JOBS=24 wsdiff proto_base proto_patch +.fi +.PP +Note that this variable is also used for nightly(1) so when run from +nightly(1), wsdiff will honor the setting. .LP .SH SEE ALSO .LP
--- a/usr/src/tools/scripts/wsdiff.py Thu Jul 22 10:15:06 2010 +0800 +++ b/usr/src/tools/scripts/wsdiff.py Thu Jul 22 10:19:00 2010 +0200 @@ -1,4 +1,4 @@ -#!/usr/bin/python2.4 +#!/usr/bin/python2.6 # # CDDL HEADER START # @@ -19,8 +19,7 @@ # # CDDL HEADER END # -# Copyright 2009 Sun Microsystems, Inc. All rights reserved. -# Use is subject to license terms. +# Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. # # @@ -70,14 +69,27 @@ # -r Log results and observed differences # -i Tell wsdiff which objects to compare via an input file list -import datetime, fnmatch, getopt, profile, os, popen2, commands -import re, select, string, struct, sys, tempfile, time +import datetime, fnmatch, getopt, os, profile, commands +import re, resource, select, shutil, signal, string, struct, sys, tempfile +import time, threading from stat import * # Human readable diffs truncated by default if longer than this # Specifying -v on the command line will override diffs_sz_thresh = 4096 +# Lock name Provides exclusive access to +# --------------+------------------------------------------------ +# output_lock standard output or temporary file (difference()) +# log_lock the results file (log_difference()) +# wset_lock changedFiles list (workerThread()) +output_lock = threading.Lock() +log_lock = threading.Lock() +wset_lock = threading.Lock() + +# Variable for thread control +keep_processing = True + # Default search path for wsdiff wsdiff_path = [ "/usr/bin", "/usr/ccs/bin", @@ -103,30 +115,57 @@ # Logging routines # +# Debug message to be printed to the screen, and the log file +def debug(msg) : + + # Add prefix to highlight debugging message + msg = "## " + msg + if debugon : + output_lock.acquire() + print >> sys.stdout, msg + sys.stdout.flush() + output_lock.release() + if logging : + log_lock.acquire() + print >> log, msg + log.flush() + log_lock.release() + # Informational message to be printed to the screen, and the log file def info(msg) : + output_lock.acquire() print >> sys.stdout, msg + sys.stdout.flush() + output_lock.release() if logging : + log_lock.acquire() print >> log, msg - sys.stdout.flush() + log.flush() + log_lock.release() # Error message to be printed to the screen, and the log file def error(msg) : - + + output_lock.acquire() print >> sys.stderr, "ERROR:", msg sys.stderr.flush() + output_lock.release() if logging : + log_lock.acquire() print >> log, "ERROR:", msg log.flush() + log_lock.release() # Informational message to be printed only to the log, if there is one. def v_info(msg) : if logging : + log_lock.acquire() print >> log, msg log.flush() - + log_lock.release() + # # Flag a detected file difference # Display the fileName to stdout, and log the difference @@ -136,8 +175,13 @@ if f in wsdiff_exceptions : return - print >> sys.stdout, f - sys.stdout.flush() + output_lock.acquire() + if sorted : + differentFiles.append(f) + else: + print >> sys.stdout, f + sys.stdout.flush() + output_lock.release() log_difference(f, dtype, diffs) @@ -145,7 +189,9 @@ # Do the actual logging of the difference to the results file # def log_difference(f, dtype, diffs) : + if logging : + log_lock.acquire() print >> log, f print >> log, "NOTE:", dtype, "difference detected." @@ -162,6 +208,7 @@ print >> log, diffs print >> log, "\n" log.flush() + log_lock.release() ##### @@ -173,37 +220,60 @@ # def diffFileData(tmpf1, tmpf2) : + binaries = False + # Filter the data through od(1) if the data is detected # as being binary if isBinary(tmpf1) or isBinary(tmpf2) : + binaries = True tmp_od1 = tmpf1 + ".od" tmp_od2 = tmpf2 + ".od" - + cmd = od_cmd + " -c -t x4" + " " + tmpf1 + " > " + tmp_od1 os.system(cmd) cmd = od_cmd + " -c -t x4" + " " + tmpf2 + " > " + tmp_od2 os.system(cmd) - + tmpf1 = tmp_od1 tmpf2 = tmp_od2 - data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2) + try: + data = commands.getoutput(diff_cmd + " " + tmpf1 + " " + tmpf2) + # Remove the temp files as we no longer need them. + if binaries : + try: + os.unlink(tmp_od1) + except OSError, e: + error("diffFileData: unlink failed %s" % e) + try: + os.unlink(tmp_od2) + except OSError, e: + error("diffFileData: unlink failed %s" % e) + except: + error("failed to get output of command: " + diff_cmd + " " \ + + tmpf1 + " " + tmpf2) + + # Send exception for the failed command up + raise + return return data # # Return human readable diffs betweeen two datasets # -def diffData(d1, d2) : +def diffData(base, ptch, d1, d2) : - global tmpFile1 - global tmpFile2 + t = threading.currentThread() + tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName() + tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() try: fd1 = open(tmpFile1, "w") except: error("failed to open: " + tmpFile1) cleanup(1) + try: fd2 = open(tmpFile2, "w") except: @@ -253,11 +323,13 @@ # def usage() : sys.stdout.flush() - print >> sys.stderr, """Usage: wsdiff [-vVt] [-r results ] [-i filelist ] old new + print >> sys.stderr, """Usage: wsdiff [-dvVst] [-r results ] [-i filelist ] old new + -d Print debug messages about the progress -v Do not truncate observed diffs in results -V Log *all* ELF sect diffs vs. logging the first diff found -t Use onbld tools in $SRC/tools -r Log results and observed differences + -s Produce sorted list of differences -i Tell wsdiff which objects to compare via an input file list""" sys.exit(1) @@ -266,11 +338,13 @@ # def args() : + global debugon global logging global vdiffs global reportAllSects + global sorted - validOpts = 'i:r:vVt?' + validOpts = 'di:r:vVst?' baseRoot = "" ptchRoot = "" @@ -293,11 +367,15 @@ usage(); for opt,val in optlist : - if opt == '-i' : + if opt == '-d' : + debugon = True + elif opt == '-i' : fileNamesFile = val elif opt == '-r' : results = val logging = True + elif opt == '-s' : + sorted = True elif opt == '-v' : vdiffs = True elif opt == '-V' : @@ -343,7 +421,6 @@ 'jar' : 'Java Archive', 'html' : 'HTML', 'ln' : 'Lint Library', - 'esa' : 'Elfsign Activation', 'db' : 'Sqlite Database' } try: @@ -420,51 +497,67 @@ # a list of deleted files (files found only in base) # def protoCatalog(base, ptch) : + compFiles = [] # List of files in both proto areas ptchList = [] # List of file in patch proto area newFiles = [] # New files detected deletedFiles = [] # Deleted files + debug("Getting the list of files in the base area"); baseFilesList = list(findFiles(base)) baseStringLength = len(base) - + debug("Found " + str(len(baseFilesList)) + " files") + + debug("Getting the list of files in the patch area"); ptchFilesList = list(findFiles(ptch)) ptchStringLength = len(ptch) + debug("Found " + str(len(ptchFilesList)) + " files") # Inventory files in the base proto area + debug("Determining the list of regular files in the base area"); for fn in baseFilesList : if os.path.islink(fn) : continue fileName = fn[baseStringLength:] compFiles.append(fileName) + debug("Found " + str(len(compFiles)) + " files") # Inventory files in the patch proto area + debug("Determining the list of regular files in the patch area"); for fn in ptchFilesList : if os.path.islink(fn) : continue fileName = fn[ptchStringLength:] ptchList.append(fileName) + debug("Found " + str(len(ptchList)) + " files") # Deleted files appear in the base area, but not the patch area + debug("Searching for deleted files by comparing the lists") for fileName in compFiles : if not fileName in ptchList : deletedFiles.append(fileName) + debug("Found " + str(len(deletedFiles)) + " deleted files") # Eliminate "deleted" files from the list of objects appearing # in both the base and patch proto areas + debug("Eliminating deleted files from the list of objects") for fileName in deletedFiles : try: compFiles.remove(fileName) except: error("filelist.remove() failed") + debug("List for comparison reduced to " + str(len(compFiles)) \ + + " files") # New files appear in the patch area, but not the base + debug("Getting the list of newly added files") for fileName in ptchList : if not fileName in compFiles : newFiles.append(fileName) + debug("Found " + str(len(newFiles)) + " new files") return compFiles, newFiles, deletedFiles @@ -488,6 +581,7 @@ files = [] files = fd.readlines() + fd.close() for f in files : ptch_present = True @@ -534,9 +628,11 @@ elif ptch_present : newFiles.append(fn) else : - if os.path.islink(base + fn) and os.path.islink(ptch + fn) : + if os.path.islink(base + fn) and \ + os.path.islink(ptch + fn) : continue - error(f + " in file list, but not in either tree. Skipping...") + error(f + " in file list, but not in either tree. " + \ + "Skipping...") return compFiles, newFiles, deletedFiles @@ -582,6 +678,7 @@ if len(hstring) == 0 : error("Failed to dump ELF header for " + f) + raise return # elfdump(1) dumps the section headers with the section name @@ -609,7 +706,9 @@ data = commands.getoutput(dump_cmd + " -sn " + section + " " + f) if len(data) == 0 : - error(cmd + " yielded no data") + error(dump_cmd + "yielded no data on section " + section + \ + " of " + f) + raise return # dump(1) displays the file name to start... @@ -629,6 +728,10 @@ text_sections = [ '.text', '.init', '.fini' ] def diff_elf_section(f1, f2, section, sh_type) : + t = threading.currentThread() + tmpFile1 = tmpDir1 + os.path.basename(f1) + t.getName() + tmpFile2 = tmpDir2 + os.path.basename(f2) + t.getName() + if (sh_type == "SHT_RELA") : # sh_type == SHT_RELA cmd1 = elfdump_cmd + " -r " + f1 + " > " + tmpFile1 cmd2 = elfdump_cmd + " -r " + f2 + " > " + tmpFile2 @@ -651,8 +754,10 @@ cmd1 = elfdump_cmd + " -i " + f1 + " > " + tmpFile1 cmd2 = elfdump_cmd + " -i " + f2 + " > " + tmpFile2 elif (section == ".symtab" or section == ".dynsym") : - cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + " > " + tmpFile1 - cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + " > " + tmpFile2 + cmd1 = elfdump_cmd + " -s -N " + section + " " + f1 + \ + " > " + tmpFile1 + cmd2 = elfdump_cmd + " -s -N " + section + " " + f2 + \ + " > " + tmpFile2 elif (section in text_sections) : # dis sometimes complains when it hits something it doesn't # know how to disassemble. Just ignore it, as the output @@ -673,6 +778,16 @@ data = diffFileData(tmpFile1, tmpFile2) + # remove temp files as we no longer need them + try: + os.unlink(tmpFile1) + except OSError, e: + error("diff_elf_section: unlink failed %s" % e) + try: + os.unlink(tmpFile2) + except OSError, e: + error("diff_elf_section: unlink failed %s" % e) + return (data) # @@ -711,10 +826,16 @@ global logging - base_header = get_elfheader(base) + try: + base_header = get_elfheader(base) + except: + return sections = base_header.keys() - ptch_header = get_elfheader(ptch) + try: + ptch_header = get_elfheader(ptch) + except: + return e2_only_sections = ptch_header.keys() e1_only_sections = [] @@ -732,32 +853,33 @@ if len(e1_only_sections) > 0 : if quiet : return 1 - info(fileName); - if not logging : - return 1 - slist = "" - for sect in e1_only_sections : - slist = slist + sect + "\t" - v_info("\nELF sections found in " + \ - base + " but not in " + ptch) - v_info("\n" + slist) + data = "" + if logging : + slist = "" + for sect in e1_only_sections : + slist = slist + sect + "\t" + data = "ELF sections found in " + \ + base + " but not in " + ptch + \ + "\n\n" + slist + + difference(fileName, "ELF", data) return 1 - + if len(e2_only_sections) > 0 : if quiet : return 1 - - info(fileName); - if not logging : - return 1 + + data = "" + if logging : + slist = "" + for sect in e2_only_sections : + slist = slist + sect + "\t" + data = "ELF sections found in " + \ + ptch + " but not in " + base + \ + "\n\n" + slist - slist = "" - for sect in e2_only_sections : - slist = slist + sect + "\t" - v_info("\nELF sections found in " + \ - ptch + " but not in " + base) - v_info("\n" + slist) + difference(fileName, "ELF", data) return 1 # Look for preferred sections, and put those at the @@ -774,30 +896,54 @@ if sect in sections_to_skip : continue - s1 = extract_elf_section(base, sect); - s2 = extract_elf_section(ptch, sect); + try: + s1 = extract_elf_section(base, sect); + except: + return + + try: + s2 = extract_elf_section(ptch, sect); + except: + return if len(s1) != len (s2) or s1 != s2: if not quiet: sh_type = base_header[sect] - data = diff_elf_section(base, ptch, sect, \ - sh_type) + data = diff_elf_section(base, ptch, \ + sect, sh_type) # If all ELF sections are being reported, then # invoke difference() to flag the file name to # stdout only once. Any other section differences # should be logged to the results file directly if not first_section : - log_difference(fileName, "ELF " + sect, data) + log_difference(fileName, \ + "ELF " + sect, data) else : - difference(fileName, "ELF " + sect, data) + difference(fileName, "ELF " + sect, \ + data) if not reportAllSects : return 1 first_section = False + return 0 ##### +# recursively remove 2 directories +# +# Used for removal of temporary directory strucures (ignores any errors). +# +def clearTmpDirs(dir1, dir2) : + + if os.path.isdir(dir1) > 0 : + shutil.rmtree(dir1, True) + + if os.path.isdir(dir2) > 0 : + shutil.rmtree(dir2, True) + + +##### # Archive object comparison # # Returns 1 if difference detected @@ -807,89 +953,100 @@ def compareArchives(base, ptch, fileType) : fileName = fnFormat(base) - - # clear the temp directories - baseCmd = "rm -rf " + tmpDir1 + "*" - status, output = commands.getstatusoutput(baseCmd) - if status != 0 : - error(baseCmd + " failed: " + output) - return -1 - - ptchCmd = "rm -rf " + tmpDir2 + "*" - status, output = commands.getstatusoutput(ptchCmd) - if status != 0 : - error(ptchCmd + " failed: " + output) - return -1 + t = threading.currentThread() + ArchTmpDir1 = tmpDir1 + os.path.basename(base) + t.getName() + ArchTmpDir2 = tmpDir2 + os.path.basename(base) + t.getName() # # Be optimistic and first try a straight file compare # as it will allow us to finish up quickly. + # if compareBasic(base, ptch, True, fileType) == 0 : return 0 + try: + os.makedirs(ArchTmpDir1) + except OSError, e: + error("compareArchives: makedir failed %s" % e) + return -1 + try: + os.makedirs(ArchTmpDir2) + except OSError, e: + error("compareArchives: makedir failed %s" % e) + return -1 + # copy over the objects to the temp areas, and # unpack them - baseCmd = "cp -fp " + base + " " + tmpDir1 + baseCmd = "cp -fp " + base + " " + ArchTmpDir1 status, output = commands.getstatusoutput(baseCmd) if status != 0 : error(baseCmd + " failed: " + output) + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 - ptchCmd = "cp -fp " + ptch + " " + tmpDir2 + ptchCmd = "cp -fp " + ptch + " " + ArchTmpDir2 status, output = commands.getstatusoutput(ptchCmd) if status != 0 : error(ptchCmd + " failed: " + output) + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 bname = string.split(fileName, '/')[-1] if fileType == "Java Archive" : - baseCmd = "cd " + tmpDir1 + "; " + "jar xf " + bname + \ + baseCmd = "cd " + ArchTmpDir1 + "; " + "jar xf " + bname + \ "; rm -f " + bname + " META-INF/MANIFEST.MF" - ptchCmd = "cd " + tmpDir2 + "; " + "jar xf " + bname + \ + ptchCmd = "cd " + ArchTmpDir2 + "; " + "jar xf " + bname + \ "; rm -f " + bname + " META-INF/MANIFEST.MF" elif fileType == "ELF Object Archive" : - baseCmd = "cd " + tmpDir1 + "; " + "/usr/ccs/bin/ar x " + \ + baseCmd = "cd " + ArchTmpDir1 + "; " + "/usr/ccs/bin/ar x " + \ bname + "; rm -f " + bname - ptchCmd = "cd " + tmpDir2 + "; " + "/usr/ccs/bin/ar x " + \ + ptchCmd = "cd " + ArchTmpDir2 + "; " + "/usr/ccs/bin/ar x " + \ bname + "; rm -f " + bname else : error("unexpected file type: " + fileType) + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return -1 os.system(baseCmd) os.system(ptchCmd) - baseFlist = list(findFiles(tmpDir1)) - ptchFlist = list(findFiles(tmpDir2)) + baseFlist = list(findFiles(ArchTmpDir1)) + ptchFlist = list(findFiles(ArchTmpDir2)) # Trim leading path off base/ptch file lists flist = [] for fn in baseFlist : - flist.append(str_prefix_trunc(fn, tmpDir1)) + flist.append(str_prefix_trunc(fn, ArchTmpDir1)) baseFlist = flist flist = [] for fn in ptchFlist : - flist.append(str_prefix_trunc(fn, tmpDir2)) + flist.append(str_prefix_trunc(fn, ArchTmpDir2)) ptchFlist = flist for fn in ptchFlist : if not fn in baseFlist : difference(fileName, fileType, \ fn + " added to " + fileName) + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 for fn in baseFlist : if not fn in ptchFlist : difference(fileName, fileType, \ fn + " removed from " + fileName) + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 - differs = compareOneFile((tmpDir1 + fn), (tmpDir2 + fn), True) + differs = compareOneFile((ArchTmpDir1 + fn), \ + (ArchTmpDir2 + fn), True) if differs : difference(fileName, fileType, \ fn + " in " + fileName + " differs") + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 1 + + clearTmpDirs(ArchTmpDir1, ArchTmpDir2) return 0 ##### @@ -947,7 +1104,7 @@ return 1 else : if len(baseData) != len(ptchData) or baseData != ptchData : - diffs = diffData(baseData, ptchData) + diffs = diffData(base, ptch, baseData, ptchData) difference(fileName, fileType, diffs) return 1 return 0 @@ -964,18 +1121,25 @@ def compareByDumping(base, ptch, quiet, fileType) : fileName = fnFormat(base); + t = threading.currentThread() + tmpFile1 = tmpDir1 + os.path.basename(base) + t.getName() + tmpFile2 = tmpDir2 + os.path.basename(ptch) + t.getName() if fileType == "Lint Library" : baseCmd = lintdump_cmd + " -ir " + base + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + " > " + tmpFile1 + " | egrep -v '(LINTOBJ|LINTMOD):'" + \ + " | grep -v PASS[1-3]:" + \ + " > " + tmpFile1 ptchCmd = lintdump_cmd + " -ir " + ptch + \ - " | egrep -v '(LINTOBJ|LINTMOD):'" + " > " + tmpFile2 + " | egrep -v '(LINTOBJ|LINTMOD):'" + \ + " | grep -v PASS[1-3]:" + \ + " > " + tmpFile2 elif fileType == "Sqlite Database" : baseCmd = "echo .dump | " + sqlite_cmd + base + " > " + \ tmpFile1 ptchCmd = "echo .dump | " + sqlite_cmd + ptch + " > " + \ tmpFile2 - + os.system(baseCmd) os.system(ptchCmd) @@ -983,10 +1147,12 @@ baseFile = open(tmpFile1) except: error("could not open: " + tmpFile1) + return try: ptchFile = open(tmpFile2) except: error("could not open: " + tmpFile2) + return baseData = baseFile.read() ptchData = ptchFile.read() @@ -997,36 +1163,71 @@ if len(baseData) != len(ptchData) or baseData != ptchData : if not quiet : data = diffFileData(tmpFile1, tmpFile2); + try: + os.unlink(tmpFile1) + except OSError, e: + error("compareByDumping: unlink failed %s" % e) + try: + os.unlink(tmpFile2) + except OSError, e: + error("compareByDumping: unlink failed %s" % e) difference(fileName, fileType, data) return 1 + + # Remove the temporary files now. + try: + os.unlink(tmpFile1) + except OSError, e: + error("compareByDumping: unlink failed %s" % e) + try: + os.unlink(tmpFile2) + except OSError, e: + error("compareByDumping: unlink failed %s" % e) + return 0 ##### -# Compare two elfsign activation files. This ignores the activation -# files themselves and reports a difference if and only if the -# corresponding base files are different. +# +# SIGINT signal handler. Changes thread control variable to tell the threads +# to finish their current job and exit. # -# Returns 1 if difference detected -# 0 if no difference detected -# -1 on error -# -def compareActivation(base, ptch, quiet, fileType) : +def discontinue_processing(signl, frme): + global keep_processing - fileName = fnFormat(base) + print >> sys.stderr, "Caught Ctrl-C, stopping the threads" + keep_processing = False + + return 0 - # Drop the .esa suffix from both filenames. - base = base[0:base.rfind('.esa')] - ptch = ptch[0:ptch.rfind('.esa')] +##### +# +# worker thread for changedFiles processing +# +class workerThread(threading.Thread) : + def run(self): + global wset_lock + global changedFiles + global baseRoot + global ptchRoot + global keep_processing - result = compareOneFile(base, ptch, True) - if result == -1 : - error("unable to compare " + fileName) - elif result == 1 : - if not quiet : - difference(fileName, fileType, \ - "change in corresponding ELF file") + while (keep_processing) : + # grab the lock to changedFiles and remove one member + # and process it + wset_lock.acquire() + try : + fn = changedFiles.pop() + except IndexError : + # there is nothing more to do + wset_lock.release() + return + wset_lock.release() - return result + base = baseRoot + fn + ptch = ptchRoot + fn + + compareOneFile(base, ptch, False) + ##### # Compare two objects. Detect type changes. @@ -1067,9 +1268,6 @@ elif ( fileType == 'Sqlite Database' ) : return compareByDumping(base, ptch, quiet, fileType) - elif ( fileType == 'Elfsign Activation' ) : - return compareActivation(base, ptch, quiet, fileType) - else : # it has to be some variety of text file return compareBasic(base, ptch, quiet, fileType) @@ -1077,14 +1275,13 @@ # Cleanup and self-terminate def cleanup(ret) : - if len(tmpDir1) > 0 and len(tmpDir2) > 0 : - - baseCmd = "rm -rf " + tmpDir1 - ptchCmd = "rm -rf " + tmpDir2 - - os.system(baseCmd) - os.system(ptchCmd) - + debug("Performing cleanup (" + str(ret) + ")") + if os.path.isdir(tmpDir1) > 0 : + shutil.rmtree(tmpDir1) + + if os.path.isdir(tmpDir2) > 0 : + shutil.rmtree(tmpDir2) + if logging : log.close() @@ -1099,7 +1296,7 @@ global logging, vdiffs, reportAllSects # Named temporary files / directories - global tmpDir1, tmpDir2, tmpFile1, tmpFile2 + global tmpDir1, tmpDir2 # Command paths global lintdump_cmd, elfdump_cmd, dump_cmd, dis_cmd, od_cmd, diff_cmd, sqlite_cmd @@ -1110,8 +1307,20 @@ # Essentially "uname -p" global arch + # changed files for worker thread processing + global changedFiles + global baseRoot + global ptchRoot + + # Sort the list of files from a temporary file + global sorted + global differentFiles + + # Debugging indicator + global debugon + # Some globals need to be initialized - logging = vdiffs = reportAllSects = False + debugon = logging = vdiffs = reportAllSects = sorted = False # Process command line arguments @@ -1139,7 +1348,11 @@ v_info("# This file was produced by wsdiff") v_info(dateTimeStr) - # + # Changed files (used only for the sorted case) + if sorted : + differentFiles = [] + + # # Build paths to the tools required tools # # Try to look for tools in $SRC/tools if the "-t" option @@ -1164,6 +1377,18 @@ sqlite_cmd = find_tool("sqlite") # + # Set resource limit for number of open files as high as possible. + # This might get handy with big number of threads. + # + (nofile_soft, nofile_hard) = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, + (nofile_hard, nofile_hard)) + except: + error("cannot set resource limits for number of open files") + sys.exit(1) + + # # validate the base and patch paths # if baseRoot[-1] != '/' : @@ -1195,13 +1420,14 @@ pid = os.getpid() tmpDir1 = "/tmp/wsdiff_tmp1_" + str(pid) + "/" tmpDir2 = "/tmp/wsdiff_tmp2_" + str(pid) + "/" - if not os.path.exists(tmpDir1) : + try: os.makedirs(tmpDir1) - if not os.path.exists(tmpDir2) : + except OSError, e: + error("main: makedir failed %s" % e) + try: os.makedirs(tmpDir2) - - tmpFile1 = tmpDir1 + "f1" - tmpFile2 = tmpDir2 + "f2" + except OSError, e: + error("main: makedir failed %s" % e) # Derive a catalog of new, deleted, and to-be-compared objects # either from the specified base and patch proto areas, or from @@ -1212,12 +1438,15 @@ changedFiles, newFiles, deletedFiles = \ flistCatalog(baseRoot, ptchRoot, fileNamesFile) else : - changedFiles, newFiles, deletedFiles = protoCatalog(baseRoot, ptchRoot) + changedFiles, newFiles, deletedFiles = \ + protoCatalog(baseRoot, ptchRoot) if len(newFiles) > 0 : newOrDeleted = True info("\nNew objects found: ") + if sorted : + newFiles.sort() for fn in newFiles : info(fnFormat(fn)) @@ -1225,12 +1454,15 @@ newOrDeleted = True info("\nObjects removed: ") + if sorted : + deletedFiles.sort() for fn in deletedFiles : info(fnFormat(fn)) if newOrDeleted : - info("\nChanged objects: "); - + info("\nChanged objects: ") + if sorted : + debug("The list will appear after the processing is done") # Here's where all the heavy lifting happens # Perform a comparison on each object appearing in @@ -1238,11 +1470,53 @@ # file types of each object, and will vector off to # the appropriate comparison routine, where the compare # will happen, and any differences will be reported / logged - for fn in changedFiles : - base = baseRoot + fn - ptch = ptchRoot + fn + + # determine maximum number of worker threads by using + # DMAKE_MAX_JOBS environment variable set by nightly(1) + # or get number of CPUs in the system + try: + max_threads = int(os.environ['DMAKE_MAX_JOBS']) + except: + max_threads = os.sysconf("SC_NPROCESSORS_ONLN") + # If we cannot get number of online CPUs in the system + # run unparallelized otherwise bump the number up 20% + # to achieve best results. + if max_threads == -1 : + max_threads = 1 + else : + max_threads += max_threads/5 + + # Set signal handler to attempt graceful exit + debug("Setting signal handler") + signal.signal( signal.SIGINT, discontinue_processing ) - compareOneFile(base, ptch, False) + # Create and unleash the threads + # Only at most max_threads must be running at any moment + mythreads = [] + debug("Spawning " + str(max_threads) + " threads"); + for i in range(max_threads) : + thread = workerThread() + mythreads.append(thread) + mythreads[i].start() + + # Wait for the threads to finish and do cleanup if interrupted + debug("Waiting for the threads to finish") + while True: + if not True in [thread.isAlive() for thread in mythreads]: + break + else: + # Some threads are still going + time.sleep(1) + + # Interrupted by SIGINT + if keep_processing == False : + cleanup(1) + + # If the list of differences was sorted it is stored in an array + if sorted : + differentFiles.sort() + for f in differentFiles : + info(fnFormat(f)) # We're done, cleanup. cleanup(0)