Source Code for Module csb.build

  1  """ 
  2  CSB build related tools and programs. 
  3   
  4  When executed as a program, this module will run the CSB Build Console and 
  5  build the source tree it belongs to. The source tree is added at the 
  6  B{beginning} of sys.path to make sure that all subsequent imports from the 
  7  Test and Doc consoles will import the right thing (think of multiple CSB 
  8  packages installed on the same server). 
  9   
 10  Here is how to build, test and package the whole project:: 
 11   
 12      $ hg clone https://hg.codeplex.com/csb CSB 
 13      $ CSB/csb/build.py -o <output directory> 
 14   
 15  The Console can also be imported and instantiated as a regular Python class. 
 16  In this case the Console again builds the source tree it is part of, but 
 17  sys.path will remain intact. Therefore, the Console will assume that all 
 18  modules currently in memory, as well as those that can be subsequently imported 
 19  by the Console itself, belong to the same CSB package. 
 20   
 21  @note: The CSB build services no longer support the option to build external 
 22         source trees. 
 23  @see: [CSB 0000038] 
 24  """ 
 25  from __future__ import print_function 
 26   
 27  import os 
 28  import sys 
 29  import getopt 
 30  import traceback 
 31  import compileall 
 32           
 33  if os.path.basename(__file__) == '__init__.py': 
 34      PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) 
 35  else: 
 36      PARENT = os.path.abspath(os.path.dirname(__file__)) 
 37   
 38  ROOT = 'csb' 
 39  SOURCETREE = os.path.abspath(os.path.join(PARENT, "..")) 
 40   
 41  if __name__ == '__main__': 
 42   
 43      # make sure "import io" imports the built in module, not csb.io 
 44      # io is required by tarfile     
 45      for path in sys.path: 
 46          if path.startswith(SOURCETREE): 
 47              sys.path.remove(path) 
 48               
 49      import io 
 50      assert hasattr(io, 'BufferedIOBase')    
 51               
 52      sys.path = [SOURCETREE] + sys.path 
 53   
 54   
 55  """ 
 56  It is now safe to import any modules   
 57  """ 
 58  import imp 
 59  import shutil 
 60  import tarfile 
 61   
 62  import csb 
 63   
 64  from abc import ABCMeta, abstractmethod 
 65  from csb.io import Shell 
 66   
 67   
 68 -class BuildTypes(object): 
 69      """ 
 70      Enumeration of build types. 
 71      """ 
 72       
 73      SOURCE = 'source' 
 74      BINARY = 'binary' 
 75   
 76      _du = { SOURCE: 'sdist', BINARY: 'bdist' }     
 77       
 78      @staticmethod 
 79 -    def get(key): 
 80          try: 
 81              return BuildTypes._du[key] 
 82          except KeyError: 
 83              raise ValueError('Unhandled build type: {0}'.format(key)) 
 84       
 85           
 86 -class Console(object): 
 87      """ 
 88      CSB Build Bot. Run with -h for usage. 
 89       
 90      @param output: build output directory 
 91      @type output: str 
 92      @param verbosity: verbosity level 
 93      @type verbosity: int 
 94       
 95      @note: The build console automatically detects and builds the csb package 
 96             it belongs to. You cannot build a different source tree with it. 
 97             See the module documentation for more info. 
 98      """ 
 99       
100      PROGRAM = __file__ 
101       
102      USAGE = r""" 
103  CSB Build Console: build, test and package the entire csb project. 
104   
105  Usage: 
106       python {program} -o output [-v verbosity] [-t type] [-h] 
107        
108  Options: 
109        -o  output     Build output directory 
110        -v  verbosity  Verbosity level, default is 1 
111        -t  type       Build type: 
112                          source - build source code distribution (default) 
113                          binary - build executable 
114        -h, --help     Display this help 
115      """     
116       
117 -    def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE): 
118           
119          self._input = None 
120          self._output = None 
121          self._temp = None 
122          self._docs = None          
123          self._apidocs = None 
124          self._root = None 
125          self._verbosity = None 
126          self._type = buildtype 
127          self._dist = BuildTypes.get(buildtype)  
128               
129          if os.path.join(SOURCETREE, ROOT) != PARENT: 
130              raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT)) 
131          self._input = SOURCETREE 
132           
133          self._success = True 
134                   
135          self.output = output 
136          self.verbosity = verbosity 
137           
138      @property 
139 -    def input(self): 
140          return self._input         
141           
142      @property 
143 -    def output(self): 
144          return self._output 
145      @output.setter 
146 -    def output(self, value): 
147          #value = os.path.dirname(value)         
148          self._output = os.path.abspath(value) 
149          self._temp = os.path.join(self._output, 'build') 
150          self._docs = os.path.join(self._temp, 'docs')          
151          self._apidocs = os.path.join(self._docs, 'api') 
152          self._root = os.path.join(self._temp, ROOT)     
153   
154      @property 
155 -    def verbosity(self): 
156          return self._verbosity 
157      @verbosity.setter 
158 -    def verbosity(self, value): 
159          self._verbosity = int(value)                      
160   
161 -    def build(self): 
162          """ 
163          Run the console. 
164          """ 
165          self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE)) 
166           
167          self._init()         
168          v = self._revision() 
169          self._doc(v) 
170          self._test() 
171           
172          self._compile()         
173          vn = self._package()      
174           
175          if self._success: 
176              self.log('\n# Done ({0}).\n'.format(vn.full)) 
177              return True 
178   
179          self.log('\n# Build failed.\n') 
180          return False 
181   
182 -    def log(self, message, level=1, ending='\n'): 
183   
184          if self._verbosity >= level:              
185              sys.stdout.write(message) 
186              sys.stdout.write(ending) 
187          sys.stdout.flush() 
188           
189 -    def _init(self): 
190          """ 
191          Collect all required stuff in the output folder. 
192          """ 
193          self.log('# Preparing the file system...') 
194   
195          if not os.path.exists(self._output): 
196              self.log('Creating output directory {0}'.format(self._output), level=2) 
197              os.mkdir(self._output) 
198   
199          if os.path.exists(self._temp): 
200              self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)          
201              shutil.rmtree(self._temp) 
202                       
203          self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)             
204          shutil.copytree(self._input, self._temp) 
205                       
206          if os.path.exists(self._apidocs): 
207              self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)             
208              shutil.rmtree(self._apidocs) 
209          if not os.path.isdir(self._docs): 
210              self.log('Creating docs directory {0}'.format(self._docs), level=2) 
211              os.mkdir(self._docs)             
212          self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)                         
213          os.mkdir(self._apidocs) 
214           
215 -    def _revision(self): 
216          """ 
217          Write the actual revision number to L{ROOT}.__version__ 
218          """ 
219          self.log('\n# Setting the most recent Revision Number...')         
220          root = os.path.join(self._root, '__init__.py') 
221           
222          self.log('Retrieving revision number from {0}'.format(root), level=2)                
223          rh = GitHandler(root)         
224          revision = rh.read().revision 
225           
226          self.log('Writing back revision number {0}'.format(revision), level=2)         
227          version = rh.write(revision, root) 
228   
229          self.log('  This is {0}.__version__ {1}'.format(ROOT, version), level=1) 
230          csb.__version__ = version 
231           
232          return version  
233                   
234 -    def _test(self): 
235          """ 
236          Run tests. Also make sure the current environment loads all modules from 
237          the input folder. 
238          """ 
239          import csb.test 
240          assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!'     #@UndefinedVariable 
241           
242          from csb.test import unittest 
243                           
244          newdata = os.path.join(self._temp, ROOT, 'test', 'data') 
245          csb.test.Config.setDefaultDataRoot(newdata) 
246          csb.test.Config.setDefaultGeneratedDataRoot(newdata) 
247   
248          self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2) 
249          csb.test.Config().ensureDataConsistency() 
250   
251          self.log('\n# Running the Test Console...') 
252                           
253          builder = csb.test.AnyTestBuilder() 
254          suite = builder.loadTests(ROOT + '.test.cases.*') 
255   
256          runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity) 
257          result = runner.run(suite) 
258          if result.wasSuccessful(): 
259              self.log('\n  Passed all unit tests') 
260          else: 
261              self.log('\n  DID NOT PASS: This build might be broken') 
262              self._success = False 
263           
264 -    def _doc(self, version): 
265          """ 
266          Build documentation in the output folder.         
267          """ 
268          self.log('\n# Generating API documentation...') 
269          try: 
270              import epydoc.cli 
271          except ImportError: 
272              self.log('\n  Skipped: epydoc is missing') 
273              return 
274                   
275          self.log('\n# Emulating ARGV for the Doc Builder...', level=2)         
276          argv = sys.argv     
277          sys.argv = ['epydoc', '--html', '-o', self._apidocs, 
278                      '--name', '{0} v{1}'.format(ROOT.upper(), version), 
279                      '--no-private', '--introspect-only', '--exclude', 'csb.test.cases', 
280                      '--css', os.path.join(self._temp, 'epydoc.css'), 
281                      '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning', 
282                      self._root] 
283           
284          if self._verbosity <= 1: 
285              sys.argv.append('-q') 
286           
287          try: 
288              epydoc.cli.cli() 
289              sys.exit(0) 
290          except SystemExit as ex: 
291              if ex.code is 0: 
292                  self.log('\n  Passed all doc tests') 
293              else: 
294                  if ex.code == 2: 
295                      self.log('\n  DID NOT PASS: Generated docs might be broken') 
296                      self._success = False 
297                  else: 
298                      self.log('\n  FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex)) 
299                      self._success = False 
300   
301          self.log('\n# Restoring the previous ARGV...', level=2)     
302          sys.argv = argv     
303           
304 -    def _compile(self): 
305          """ 
306          Byte-compile all modules and packages. 
307          """ 
308          self.log('\n# Byte-compiling all *.py files...') 
309           
310          quiet = self.verbosity <= 1 
311          valid = compileall.compile_dir(self._root, quiet=quiet, force=True) 
312           
313          if not valid: 
314              self.log('\n  FAIL: Compilation error(s)\n') 
315              self._success = False 
316                           
317 -    def _package(self): 
318          """ 
319          Make package. 
320          """ 
321          self.log('\n# Configuring CWD and ARGV for the Setup...', level=2) 
322          cwd = os.curdir 
323          os.chdir(self._temp) 
324                                   
325          if self._verbosity > 1: 
326              verbosity = '-v' 
327          else: 
328              verbosity = '-q' 
329          argv = sys.argv             
330          sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]         
331               
332          self.log('\n# Building {0} distribution...'.format(self._type)) 
333          version = package = None 
334   
335          try:        
336              setup = imp.load_source('setupcsb', 'setup.py') 
337              d = setup.build() 
338              version = setup.VERSION 
339              package = d.dist_files[0][2] 
340               
341              if self._type == BuildTypes.BINARY: 
342                  self._strip_source(package) 
343               
344          except SystemExit as ex: 
345              if ex.code is not 0: 
346                  self.log('\n  FAIL: Setup returned: \n\n{0}\n'.format(ex)) 
347                  self._success = False 
348                  package = 'FAIL' 
349               
350          self.log('\n# Restoring the previous CWD and ARGV...', level=2) 
351          os.chdir(cwd) 
352          sys.argv = argv    
353   
354          self.log('  Packaged ' + package)    
355          return version 
356       
357 -    def _strip_source(self, package, source='*.py'): 
358          """ 
359          Delete plain text source code files from the package. 
360          """     
361          cwd = os.getcwd() 
362           
363          try:   
364              tmp = os.path.join(self.output, 'tmp') 
365              os.mkdir(tmp) 
366           
367              self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)         
368              os.chdir(tmp) 
369                   
370              oldtar = tarfile.open(package, mode='r:gz') 
371              oldtar.extractall(tmp) 
372              oldtar.close() 
373               
374              newtar = tarfile.open(package, mode='w:gz')             
375       
376              try: 
377                  for i in os.walk('.'): 
378                      for fn in i[2]: 
379                          if fn.endswith('.py'): 
380                              module = os.path.join(i[0], fn); 
381                              if not os.path.isfile(module.replace('.py', '.pyc')): 
382                                  raise ValueError('Missing bytecode for module {0}'.format(module)) 
383                              else:                                           
384                                  os.remove(os.path.join(i[0], fn)) 
385                   
386                  for i in os.listdir('.'): 
387                      newtar.add(i)         
388              finally: 
389                  newtar.close() 
390                   
391          finally: 
392              self.log('\n# Restoring the previous CWD...', level=2)             
393              os.chdir(cwd) 
394              if os.path.exists(tmp): 
395                  shutil.rmtree(tmp)     
396           
397      @staticmethod 
398 -    def exit(message=None, code=0, usage=True): 
399           
400          if message: 
401              print(message) 
402          if usage: 
403              print(Console.USAGE.format(program=Console.PROGRAM))     
404                   
405          sys.exit(code)                
406   
407      @staticmethod 
408 -    def run(argv=None): 
409           
410          if argv is None: 
411              argv = sys.argv[1:] 
412               
413          output = None 
414          verb = 1 
415          buildtype = BuildTypes.SOURCE 
416                   
417          try:    
418              options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help']) 
419               
420              for option, value in options: 
421                  if option in('-h', '--help'): 
422                      Console.exit(message=None, code=0) 
423                  if option in('-o', '--output'): 
424                      if not os.path.isdir(value): 
425                          Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3) 
426                      output = value 
427                  if option in('-v', '--verbosity'): 
428                      try: 
429                          verb = int(value) 
430                      except ValueError: 
431                          Console.exit(message='E: Verbosity must be an integer.', code=4) 
432                  if option in('-t', '--type'): 
433                      if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]: 
434                          Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5) 
435                      buildtype = value                                          
436          except getopt.GetoptError as oe: 
437              Console.exit(message='E: ' + str(oe), code=1)         
438   
439          if not output: 
440              Console.exit(code=1, usage=True) 
441          else: 
442              try: 
443                  ok = Console(output, verbosity=verb, buildtype=buildtype).build() 
444                  Console.exit(code=0 if ok else 66, usage=False) 
445              except Exception as ex: 
446                  msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc()) 
447                  Console.exit(message=msg, code=99, usage=False) 
448                   
449               
450 -class RevisionError(RuntimeError): 
451       
452 -    def __init__(self, msg, code, cmd): 
453           
454          super(RevisionError, self).__init__(msg) 
455          self.code = code 
456          self.cmd = cmd 
457   
458 -class RevisionHandler(object): 
459      """ 
460      Determines the current repository revision number of a working copy. 
461       
462      @param path: a local checkout path to be examined 
463      @type path: str 
464      @param sc: name of the source control program 
465      @type sc: str  
466      """     
467       
468 -    def __init__(self, path, sc): 
469           
470          self._path = None 
471          self._sc = None 
472           
473          if os.path.exists(path): 
474              self._path = path 
475          else: 
476              raise IOError('Path not found: {0}'.format(path)) 
477          if Shell.run([sc, 'help']).code is 0: 
478              self._sc = sc 
479          else: 
480              raise RevisionError('Source control binary probe failed', None, None) 
481           
482      @property 
483 -    def path(self): 
484          return self._path 
485       
486      @property 
487 -    def sc(self): 
488          return self._sc 
489       
490      @abstractmethod 
491 -    def read(self): 
492          """ 
493          Return the current revision information. 
494          @rtype: L{RevisionInfo} 
495          """ 
496          pass 
497       
498 -    def write(self, revision, sourcefile): 
499          """ 
500          Finalize the __version__ = major.minor.micro.{revision} tag. 
501          Overwrite C{sourcefile} in place by substituting the {revision} macro. 
502           
503          @param revision: revision number to write to the source file. 
504          @type revision: int 
505          @param sourcefile: python source file with a __version__ tag, typically 
506                             "csb/__init__.py" 
507          @type sourcefile: str 
508           
509          @return: sourcefile.__version__ 
510          """ 
511          content = open(sourcefile).readlines() 
512           
513          with open(sourcefile, 'w') as src: 
514              for line in content: 
515                  if line.startswith('__version__'): 
516                      src.write(line.format(revision=revision)) 
517                  else: 
518                      src.write(line) 
519   
520          self._delcache(sourcefile) 
521          return imp.load_source('____source', sourcefile).__version__       
522       
523 -    def _run(self, cmd): 
524           
525          si = Shell.run(cmd) 
526          if si.code > 0: 
527              raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd) 
528           
529          return si.stdout.splitlines() 
530       
531 -    def _delcache(self, sourcefile): 
532           
533          compiled = os.path.splitext(sourcefile)[0] + '.pyc' 
534          if os.path.isfile(compiled): 
535              os.remove(compiled) 
536               
537          pycache = os.path.join(os.path.dirname(compiled), '__pycache__') 
538          if os.path.isdir(pycache):  
539              shutil.rmtree(pycache) 
540               
541 -class SubversionHandler(RevisionHandler): 
542   
543 -    def __init__(self, path, sc='svn'):    
544          super(SubversionHandler, self).__init__(path, sc)     
545   
546 -    def read(self): 
547   
548          cmd = '{0.sc} info {0.path} -R'.format(self) 
549          maxrevision = None 
550               
551          for line in self._run(cmd): 
552              if line.startswith('Revision:'): 
553                  rev = int(line[9:] .strip()) 
554                  if rev > maxrevision: 
555                      maxrevision = rev 
556           
557          if maxrevision is None: 
558              raise RevisionError('No revision number found', code=0, cmd=cmd) 
559           
560          return RevisionInfo(self.path, maxrevision) 
561   
562 -class MercurialHandler(RevisionHandler):   
563   
564 -    def __init__(self, path, sc='hg'): 
565           
566          if os.path.isfile(path): 
567              path = os.path.dirname(path) 
568                            
569          super(MercurialHandler, self).__init__(path, sc) 
570           
571 -    def read(self): 
572           
573          wd = os.getcwd() 
574          os.chdir(self.path) 
575           
576          try: 
577              cmd = '{0.sc} log -r tip'.format(self) 
578       
579              revision = None 
580              changeset = '' 
581               
582              for line in self._run(cmd): 
583                  if line.startswith('changeset:'): 
584                      items = line[10:].split(':') 
585                      revision = int(items[0]) 
586                      changeset = items[1].strip() 
587                      break 
588       
589              if revision is None: 
590                  raise RevisionError('No revision number found', code=0, cmd=cmd) 
591               
592              return RevisionInfo(self.path, revision, changeset)   
593           
594          finally: 
595              os.chdir(wd) 
596   
597   
598 -class GitHandler(RevisionHandler):   
599   
600 -    def __init__(self, path, sc='git'): 
601           
602          if os.path.isfile(path): 
603              path = os.path.dirname(path) 
604                            
605          super(GitHandler, self).__init__(path, sc) 
606           
607 -    def read(self): 
608           
609          wd = os.getcwd() 
610          os.chdir(self.path) 
611           
612          try: 
613              cmd = '{0.sc} rev-parse --short HEAD'.format(self) 
614              revision = None 
615               
616              for line in self._run(cmd): 
617                  if line.strip(): 
618                      revision = line.strip() 
619                      break 
620       
621              if revision is None: 
622                  raise RevisionError('No revision number found', code=0, cmd=cmd) 
623               
624              return RevisionInfo(self.path, revision, revision)   
625           
626          finally: 
627              os.chdir(wd) 
628   
629 -class RevisionInfo(object): 
630       
631 -    def __init__(self, item, revision, id=None): 
632           
633          self.item = item 
634          self.revision = revision 
635          self.id = id 
636   
637           
638 -def main(): 
639      Console.run() 
640           
641   
642  if __name__ == '__main__': 
643       
644      main() 
645