Bug#698117: unblock: rebuildd/0.4.2
Package: release.debian.org
Severity: normal
User: release.debian.org@packages.debian.org
Usertags: unblock
Please unblock package rebuildd
It appears that bug #671635 is critical and makes rebuildd not working at
all. Considering the amount of time 0.4.2 spent into unstable, I think it's
safe to push this version into testing.
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Distribution.py rebuildd-0.4.2/build/lib/rebuildd/Distribution.py
--- rebuildd-0.4.1/build/lib/rebuildd/Distribution.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Distribution.py 2008-03-05 13:21:48.000000000 +0100
@@ -0,0 +1,85 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+from RebuilddConfig import RebuilddConfig
+from RebuilddLog import RebuilddLog
+from string import Template
+
+class Distribution(object):
+ """Class implementing a Debian distribution
+
+ Substitutions are done in the command strings:
+
+ $d => The distro's name
+ $a => the target architecture
+ $p => the package's name
+ $v => the package's version
+ """
+
+ def __init__(self, name, arch):
+ self.name = name
+ self.arch = arch
+
+ def get_source_cmd(self, package):
+ """Return command used for grabing source for this distribution"""
+
+ try:
+ args = { 'd': self.name, 'a': self.arch, 'v': package.version, \
+ 'p': package.name }
+ t = Template(RebuilddConfig().get('build', 'source_cmd'))
+ return t.safe_substitute(**args)
+ except TypeError, error:
+ RebuilddLog.error("get_source_cmd has invalid format: %s" % error)
+ return None
+
+ def get_build_cmd(self, package):
+ """Return command used for building source for this distribution"""
+
+ # Strip epochs (x:) away
+ try:
+ index = package.version.index(":")
+ args = { 'd': self.name, 'a': self.arch, \
+ 'v': package.version[index+1:], 'p': package.name }
+ t = Template(RebuilddConfig().get('build', 'build_cmd'))
+ return t.safe_substitute(**args)
+ except ValueError:
+ pass
+
+ try:
+ args = { 'd': self.name, 'a': self.arch, \
+ 'v': package.version, 'p': package.name }
+ t = Template(RebuilddConfig().get('build', 'build_cmd'))
+ return t.safe_substitute(**args)
+ except TypeError, error:
+ RebuilddLog.error("get_build_cmd has invalid format: %s" % error)
+ return None
+
+ def get_post_build_cmd(self, package):
+ """Return command used after building source for this distribution"""
+
+ cmd = RebuilddConfig().get('build', 'post_build_cmd')
+ if cmd == '':
+ return None
+ try:
+ args = { 'd': self.name, 'a': self.arch, \
+ 'v': package.version, 'p': package.name }
+ t = Template(cmd)
+ return t.safe_substitute(**args)
+ except TypeError, error:
+ RebuilddLog.error("post_build_cmd has invalid format: %s" % error)
+ return None
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Dists.py rebuildd-0.4.2/build/lib/rebuildd/Dists.py
--- rebuildd-0.4.1/build/lib/rebuildd/Dists.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Dists.py 2007-12-03 11:47:42.000000000 +0100
@@ -0,0 +1,37 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+class Dists(object):
+ """Singleton implementing a set of Debian distributions"""
+
+ _instance = None
+ dists = []
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = object.__new__(cls)
+ return cls._instance
+
+ def add_dist(self, dist):
+ self.dists.append(dist)
+
+ def get_dist(self, name, arch):
+ for d in self.dists:
+ if d.name == name and d.arch == arch:
+ return d
+ return None
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Enumeration.py rebuildd-0.4.2/build/lib/rebuildd/Enumeration.py
--- rebuildd-0.4.1/build/lib/rebuildd/Enumeration.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Enumeration.py 2007-11-19 20:47:35.000000000 +0100
@@ -0,0 +1,51 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import types
+
+class Enumeration:
+ """Simple enumeration class with reverse lookup"""
+
+ def __init__(self, enumlist):
+ self.lookup = { }
+ self.reverse_lookup = { }
+ val = 0
+ for elem in enumlist:
+ if type(elem) == types.TupleType:
+ elem, val = elem
+ if type(elem) != types.StringType:
+ raise "enum name is not a string: " + elem
+ if type(val) != types.IntType:
+ raise "enum value is not an integer: " + val
+ if self.lookup.has_key(elem):
+ raise "enum name is not unique: " + elem
+ if val in self.lookup.values():
+ raise "enum value is not unique for " + val
+ self.lookup[elem] = val
+ self.reverse_lookup[val] = elem
+ val += 1
+
+ def __getattr__(self, attr):
+ if not self.lookup.has_key(attr):
+ raise AttributeError
+ return self.lookup[attr]
+
+ def whatis(self, value):
+ """Return element name for a value"""
+
+ return self.reverse_lookup[value]
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/__init__.py rebuildd-0.4.2/build/lib/rebuildd/__init__.py
--- rebuildd-0.4.1/build/lib/rebuildd/__init__.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/__init__.py 2007-11-19 20:47:35.000000000 +0100
@@ -0,0 +1,36 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+"""rebuildd - Debian packages rebuild tool"""
+
+__author__ = "Julien Danjou <acid@debian.org>"
+__cvsid__ = "$Id$"
+__version__ = "$Rev$"[6:-2]
+__all__ = [
+ "Distribution",
+ "Enumeration",
+ "Job",
+ "JobStatus",
+ "Package",
+ "Rebuildd",
+ "RebuilddConfig",
+ "RebuilddLog",
+ "RebuilddNetworkClient",
+ "RebuilddNetworkServer",
+ "RebuilddHTTPServer"
+ ]
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Job.py rebuildd-0.4.2/build/lib/rebuildd/Job.py
--- rebuildd-0.4.1/build/lib/rebuildd/Job.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Job.py 2007-12-03 11:47:42.000000000 +0100
@@ -0,0 +1,250 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+from __future__ import with_statement
+
+import threading, subprocess, smtplib, time, os, signal, socket, select
+import sqlobject
+from email.Message import Message
+from Dists import Dists
+from JobStatus import JobStatus
+from JobStatus import FailedStatus
+from RebuilddLog import RebuilddLog
+from RebuilddConfig import RebuilddConfig
+
+__version__ = "$Rev$"
+
+class Job(threading.Thread, sqlobject.SQLObject):
+ """Class implementing a build job"""
+
+ status = sqlobject.IntCol(default=JobStatus.UNKNOWN)
+ mailto = sqlobject.StringCol(default=None)
+ package = sqlobject.ForeignKey('Package', cascade=True)
+ dist = sqlobject.StringCol(default='sid')
+ arch = sqlobject.StringCol(default='any')
+ creation_date = sqlobject.DateTimeCol(default=sqlobject.DateTimeCol.now)
+ status_changed = sqlobject.DateTimeCol(default=None)
+ build_start = sqlobject.DateTimeCol(default=None)
+ build_end = sqlobject.DateTimeCol(default=None)
+ host = sqlobject.StringCol(default=None)
+ notify = None
+
+ def __init__(self, *args, **kwargs):
+ """Init job"""
+
+ threading.Thread.__init__(self)
+ sqlobject.SQLObject.__init__(self, *args, **kwargs)
+ self.do_quit = threading.Event()
+ self.status_lock = threading.Lock()
+
+ def __setattr__(self, name, value):
+ """Override setattr to log build status changes"""
+
+ if name == "status":
+ RebuilddLog.info("Job %s for %s_%s on %s/%s changed status from %s to %s"\
+ % (self.id, self.package.name, self.package.version,
+ self.dist, self.arch,
+ JobStatus.whatis(self.status),
+ JobStatus.whatis(value)))
+ self.status_changed = sqlobject.DateTimeCol.now()
+ sqlobject.SQLObject.__setattr__(self, name, value)
+
+ @property
+ def logfile(self):
+ """Compute and return logfile name"""
+
+ build_log_file = "%s/%s_%s-%s-%s-%s.%s.log" % (RebuilddConfig().get('log', 'logs_dir'),
+ self.package.name, self.package.version,
+ self.dist, self.arch,
+ self.creation_date.strftime("%Y%m%d-%H%M%S"),
+ self.id)
+ return build_log_file
+
+ def preexec_child(self):
+ """Start a new group process before executing child"""
+
+ os.setsid()
+
+ def run(self):
+ """Run job thread, download and build the package"""
+
+ self.build_start = sqlobject.DateTimeCol.now()
+
+ try:
+ with open(self.logfile, "w") as build_log:
+ build_log.write("Automatic build of %s_%s on %s for %s/%s by rebuildd %s\n" % \
+ (self.package.name, self.package.version,
+ self.host, self.dist, self.arch, __version__))
+ build_log.write("Build started at %s\n" % self.build_start)
+ build_log.write("******************************************************************************\n")
+ except IOError:
+ return
+
+ build_log = file(self.logfile, "a")
+
+ # we are building
+ with self.status_lock:
+ self.status = JobStatus.BUILDING
+
+ # execute commands
+ for cmd, failed_status in ([Dists().get_dist(self.dist, self.arch).get_source_cmd(self.package),
+ JobStatus.SOURCE_FAILED],
+ [Dists().get_dist(self.dist, self.arch).get_build_cmd(self.package),
+ JobStatus.BUILD_FAILED],
+ [Dists().get_dist(self.dist, self.arch).get_post_build_cmd(self.package),
+ JobStatus.POST_BUILD_FAILED]):
+ if cmd is None:
+ continue
+ try:
+ proc = subprocess.Popen(cmd.split(), bufsize=0,
+ preexec_fn=self.preexec_child,
+ stdout=build_log,
+ stdin=None,
+ stderr=subprocess.STDOUT)
+ except Exception, error:
+ state = 1
+ break
+ state = proc.poll()
+ while not self.do_quit.isSet() and state == None:
+ state = proc.poll()
+ self.do_quit.wait(1)
+ if self.do_quit.isSet():
+ break
+ if state != 0:
+ with self.status_lock:
+ self.status = failed_status
+ break
+
+ if self.do_quit.isSet():
+ # Kill gently the process
+ RebuilddLog.info("Killing job %s with SIGINT" % self.id)
+ try:
+ os.killpg(os.getpgid(proc.pid), signal.SIGINT)
+ except OSError, error:
+ RebuilddLog.error("Error killing job %s: %s" % (self.id, error))
+
+ # If after 60s it's not dead, KILL HIM
+ counter = 0
+ timemax = RebuilddConfig().get('build', 'kill_timeout')
+ while proc.poll() == None and counter < timemax:
+ time.sleep(1)
+ counter += 1
+ if proc.poll() == None:
+ RebuilddLog.error("Killing job %s timed out, killing with SIGKILL" \
+ % self.id)
+ os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+
+ with self.status_lock:
+ self.status = JobStatus.WAIT_LOCKED
+
+ # Reset host
+ self.host = None
+
+ build_log.write("\nBuild job killed on request\n")
+ build_log.close()
+
+ return
+
+ # build is finished
+ with self.status_lock:
+ if state == 0:
+ self.status = JobStatus.BUILD_OK
+
+ self.build_end = sqlobject.DateTimeCol.now()
+
+ build_log.write("******************************************************************************\n")
+ build_log.write("Finished with status %s at %s\n" % (JobStatus.whatis(self.status), self.build_end))
+ build_log.write("Build needed %s\n" % (self.build_end - self.build_start))
+ build_log.close()
+
+
+ # Send event to Rebuildd to inform it that it can
+ # run a brand new job!
+ if self.notify:
+ self.notify.set()
+
+ self.send_build_log()
+
+ def send_build_log(self):
+ """When job is built, send logs by mail"""
+
+ with self.status_lock:
+ if self.status != JobStatus.BUILD_OK and \
+ not self.status in FailedStatus:
+ return False
+
+ if not RebuilddConfig().getboolean('log', 'mail_successful') \
+ and self.status == JobStatus.BUILD_OK:
+ return True
+ elif not RebuilddConfig().getboolean('log', 'mail_failed') \
+ and self.status in FailedStatus:
+ return True
+
+ if self.status == JobStatus.BUILD_OK:
+ bstatus = "successful"
+ else:
+ bstatus = "failed"
+
+ msg = Message()
+ if self.mailto:
+ msg['To'] = self.mailto
+ else:
+ msg['To'] = RebuilddConfig().get('mail', 'mailto')
+ msg['From'] = RebuilddConfig().get('mail', 'from')
+ msg['Subject'] = RebuilddConfig().get('mail', 'subject_prefix') + \
+ " Log for %s build of %s_%s on %s/%s" % \
+ (bstatus,
+ self.package.name,
+ self.package.version,
+ self.dist,
+ self.arch)
+ msg['X-Rebuildd-Version'] = __version__
+ msg['X-Rebuildd-Host'] = socket.getfqdn()
+
+
+ try:
+ with open(self.logfile, "r") as build_log:
+ log = ""
+ for line in build_log.readlines():
+ log += line
+ except IOError, error:
+ RebuilddLog.error("Unable to open logfile for job %d" % self.id)
+ return False
+
+ msg.set_payload(log)
+
+ try:
+ smtp = smtplib.SMTP()
+ smtp.connect()
+ if self.mailto:
+ smtp.sendmail(RebuilddConfig().get('mail', 'from'),
+ self.mailto,
+ msg.as_string())
+ else:
+ smtp.sendmail(RebuilddConfig().get('mail', 'from'),
+ RebuilddConfig().get('mail', 'mailto'),
+ msg.as_string())
+ except Exception, error:
+ RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
+
+ return True
+
+ def __str__(self):
+ return "I: Job %s for %s_%s is status %s on %s for %s/%s" % \
+ (self.id, self.package.name, self.package.version, self.host,
+ JobStatus.whatis(self.status), self.dist, self.arch)
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/JobStatus.py rebuildd-0.4.2/build/lib/rebuildd/JobStatus.py
--- rebuildd-0.4.1/build/lib/rebuildd/JobStatus.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/JobStatus.py 2007-11-19 20:47:35.000000000 +0100
@@ -0,0 +1,35 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+from Enumeration import Enumeration
+
+JobStatus = Enumeration([ ("UNKNOWN", 0),
+ ("WAIT", 100),
+ ("WAIT_LOCKED", 150),
+ ("BUILDING", 200),
+ ("SOURCE_FAILED", 250),
+ ("BUILD_FAILED", 300),
+ ("POST_BUILD_FAILED", 350),
+ ("CANCELED", 800),
+ ("GIVEUP", 850),
+ ("FAILED", 900),
+ ("BUILD_OK", 1000) ])
+
+FailedStatus = (JobStatus.SOURCE_FAILED,
+ JobStatus.BUILD_FAILED,
+ JobStatus.POST_BUILD_FAILED)
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Package.py rebuildd-0.4.2/build/lib/rebuildd/Package.py
--- rebuildd-0.4.1/build/lib/rebuildd/Package.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Package.py 2007-12-03 11:47:42.000000000 +0100
@@ -0,0 +1,36 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import sqlobject
+
+# apt warning make me crazzzyyy
+import warnings
+warnings.filterwarnings("ignore", module='apt')
+del warnings
+import apt
+
+class Package(sqlobject.SQLObject):
+ """Class implemeting a Debian package"""
+
+ name = sqlobject.StringCol()
+ version = sqlobject.StringCol(default=None)
+ priority = sqlobject.StringCol(default=None)
+
+ @staticmethod
+ def VersionCompare(a, b):
+ return apt.VersionCompare(a.version, b.version)
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/RebuilddConfig.py rebuildd-0.4.2/build/lib/rebuildd/RebuilddConfig.py
--- rebuildd-0.4.1/build/lib/rebuildd/RebuilddConfig.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/RebuilddConfig.py 2008-06-13 20:07:00.000000000 +0200
@@ -0,0 +1,114 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import ConfigParser
+import os
+
+class RebuilddConfig(object, ConfigParser.ConfigParser):
+ """Main configuration singleton"""
+
+ config_file = "/etc/rebuildd/rebuilddrc"
+ _instance = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = object.__new__(cls)
+ cls._instance.init(*args, **kwargs)
+ return cls._instance
+
+ def init(self, dontparse=False):
+ ConfigParser.ConfigParser.__init__(self)
+ # add default sections
+ self.add_section('build')
+ self.add_section('mail')
+ self.add_section('telnet')
+ self.add_section('http')
+ self.add_section('log')
+
+ # add default values
+ self.set('build', 'check_every', '300')
+ self.set('build', 'max_threads', '2')
+ self.set('build', 'max_jobs', '5')
+ self.set('build', 'kill_timeout', '90')
+ self.set('build', 'source_cmd', 'apt-get -q --download-only -t ${d} source ${p}=${v}')
+ self.set('build', 'build_cmd', 'pbuilder build --basetgz /var/cache/pbuilder/${d}-${a}.tgz ${p}_${v}.dsc')
+ self.set('build', 'post_build_cmd', '')
+ self.set('build', 'dists', 'etch lenny sid')
+ self.set('build', 'work_dir', '/var/cache/rebuildd/build')
+ self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db')
+ self.set('build', 'build_more_recent', '1')
+ self.set('build', 'more_archs', 'any')
+
+ self.set('mail', 'from', 'rebuildd@localhost')
+ self.set('mail', 'mailto', 'rebuildd@localhost')
+ self.set('mail', 'subject_prefix', '[rebuildd]')
+
+ self.set('telnet', 'port', '9999')
+ self.set('telnet', 'ip', '127.0.0.1')
+ self.set('telnet', 'prompt', 'rebuildd@localhost->')
+ self.set('telnet', 'motd', 'Connected on rebuildd on localhost')
+
+ self.set('http', 'port', '9998')
+ self.set('http', 'ip', '0.0.0.0')
+ # This is dedicated to MadCoder
+ self.set('http', 'log_lines_nb', '30')
+ self.set('http', 'templates_dir', '/usr/share/rebuildd/templates')
+ self.set('http', 'cache', '1')
+ self.set('http', 'logfile', '/var/log/rebuildd/httpd.log')
+
+ self.set('log', 'file', "/var/log/rebuildd/rebuildd.log")
+ self.set('log', 'time_format', "%d-%m-%Y %H:%M:%S")
+ self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs")
+ self.set('log', 'mail_failed', '1')
+ self.set('log', 'mail_successful', '0')
+
+ self.arch = []
+ parch = os.popen("dpkg --print-architecture")
+ self.arch.append(parch.readline().strip())
+ parch.close()
+
+ if not dontparse:
+ self.reload()
+
+ for a in self.get('build', 'more_archs').split(' '):
+ self.arch.append(a)
+
+ def reload(self):
+ """Reload configuration file"""
+
+ return self.read(self.config_file)
+
+ def dump(self):
+ """Dump running configuration"""
+
+ conf = ""
+ for section in self.sections():
+ conf += "[" + section + "]\n"
+ for item, value in self.items(section):
+ conf += "%s = %s\n" % (item, value)
+ conf += "\n"
+ return conf
+
+ def save(self):
+ """Save configuration file"""
+
+ try:
+ self.write(file(self.config_file, 'w'))
+ except Exception, error:
+ return False
+ return True
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/RebuilddHTTPServer.py rebuildd-0.4.2/build/lib/rebuildd/RebuilddHTTPServer.py
--- rebuildd-0.4.1/build/lib/rebuildd/RebuilddHTTPServer.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/RebuilddHTTPServer.py 2009-07-27 15:51:07.000000000 +0200
@@ -0,0 +1,188 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+from __future__ import with_statement
+
+from RebuilddConfig import RebuilddConfig
+from Rebuildd import Rebuildd
+from Package import Package
+from Job import Job
+from JobStatus import JobStatus
+from JobStatus import FailedStatus
+
+import tempfile, socket, sqlobject
+import web
+import gdchart
+
+render = web.template.render(RebuilddConfig().get('http', 'templates_dir'), \
+ cache=RebuilddConfig().getboolean('http', 'cache'))
+
+class RequestIndex:
+
+ def GET(self):
+ return render.base(page=render.index(), \
+ hostname=socket.gethostname(), \
+ archs=RebuilddConfig().arch, \
+ dists=RebuilddConfig().get('build', 'dists').split(' '))
+
+class RequestPackage:
+
+ def GET(self, name=None, version=None):
+ jobs = []
+
+ if version:
+ pkg = Package.selectBy(name=name, version=version)[0]
+ title = "%s %s" % (name, version)
+ package = "%s/%s" % (name, version)
+ else:
+ pkg = Package.selectBy(name=name)[0]
+ title = package = name
+
+ jobs.extend(Job.selectBy(package=pkg))
+ return render.base(page=render.tab(jobs=jobs), \
+ hostname=socket.gethostname(), \
+ title=title, \
+ package=package, \
+ archs=RebuilddConfig().arch, \
+ dists=RebuilddConfig().get('build', 'dists').split(' '))
+
+class RequestArch:
+
+ def GET(self, dist, arch=None):
+ jobs = []
+ jobs.extend(Job.select(sqlobject.AND(Job.q.arch == arch, Job.q.dist == dist),
+ orderBy=sqlobject.DESC(Job.q.creation_date))[:10])
+ return render.base(page=render.tab(jobs=jobs), \
+ arch=arch, \
+ dist=dist, \
+ title="%s/%s" % (dist, arch), \
+ hostname=socket.gethostname(), \
+ archs=RebuilddConfig().arch, \
+ dists=RebuilddConfig().get('build', 'dists').split(' '))
+
+class RequestJob:
+
+ def GET(self, jobid=None):
+ job = Job.selectBy(id=jobid)[0]
+
+ try:
+ with open(job.logfile, "r") as build_logfile:
+ build_log = build_logfile.read()
+ except IOError, error:
+ build_log = "No build log available"
+
+ return render.base(page=render.job(job=job, build_log=build_log), \
+ hostname=socket.gethostname(), \
+ title="job %s" % job.id, \
+ archs=RebuilddConfig().arch, \
+ dists=RebuilddConfig().get('build', 'dists').split(' '))
+
+class RequestGraph:
+
+ GET = web.autodelegate("GET_")
+
+ def graph_init(self):
+ web.header("Content-Type","image/png")
+ graph = gdchart.Bar3D()
+ graph.width = 300
+ graph.height = 300
+ graph.ytitle = "Jobs"
+ graph.xtitle = "Build status"
+ graph.ext_color = [ "yellow", "orange", "red", "green"]
+ graph.bg_color = "white"
+ graph.setLabels(["WAIT", "BUILDING", "FAILED", "OK"])
+
+ return graph
+
+ def compute_stats(self, jobs):
+ jw = 0
+ jb = 0
+ jf = 0
+ jo = 0
+ for job in jobs:
+ if job.status == JobStatus.WAIT or \
+ job.status == JobStatus.WAIT_LOCKED:
+ jw += 1
+ elif job.status == JobStatus.BUILDING:
+ jb += 1
+ elif job.status in FailedStatus:
+ jf += 1
+ elif job.status == JobStatus.BUILD_OK:
+ jo += 1
+
+ return (jw, jb, jf, jo)
+
+ def GET_buildstats(self, distarch=None):
+ graph = self.graph_init()
+ if distarch == "/":
+ graph.title = "Build status"
+ jobs = Job.selectBy()
+ else:
+ dindex = distarch.rindex("/")
+ graph.title = "Build status for %s" % distarch[1:]
+ jobs = Job.selectBy(arch=distarch[dindex+1:], dist=distarch[1:dindex])
+
+ graph.setData(self.compute_stats(jobs))
+ tmp = tempfile.TemporaryFile()
+ graph.draw(tmp)
+ tmp.seek(0)
+ return tmp.read()
+
+ def GET_package(self, package=None):
+ graph = self.graph_init()
+ if package == "/":
+ graph.title = "Build status"
+ jobs = Job.selectBy()
+ else:
+ dindex = package.rindex("/")
+ graph.title = "Build status for %s" % package[1:]
+ pkg = Package.selectBy(version=package[dindex+1:], name=package[1:dindex])[0]
+ jobs = Job.selectBy(package=pkg)
+
+ graph.setData(self.compute_stats(jobs))
+ tmp = tempfile.TemporaryFile()
+ graph.draw(tmp)
+ tmp.seek(0)
+ return tmp.read()
+
+class RebuilddHTTPServer:
+ """Main HTTP server"""
+
+ urls = (
+ '/', 'RequestIndex',
+ '/dist/(.*)/arch/(.*)', 'RequestArch',
+ '/job/(.*)', 'RequestJob',
+ '/package/(.*)/(.*)', 'RequestPackage',
+ '/package/(.*)', 'RequestPackage',
+ '/graph/(.*)', 'RequestGraph',
+ )
+
+ def __init__(self):
+ Rebuildd()
+
+ def start(self):
+
+ """Run main HTTP server thread"""
+
+ web.webapi.internalerror = web.debugerror
+
+ import sys; sys.argv.append(RebuilddConfig().get('http', 'ip') + ":" + RebuilddConfig().get('http', 'port'))
+
+ app = web.application(self.urls, globals())
+ app.run()
+
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/RebuilddLog.py rebuildd-0.4.2/build/lib/rebuildd/RebuilddLog.py
--- rebuildd-0.4.1/build/lib/rebuildd/RebuilddLog.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/RebuilddLog.py 2007-11-19 20:47:35.000000000 +0100
@@ -0,0 +1,58 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import logging
+from RebuilddConfig import RebuilddConfig
+
+class RebuilddLog(object):
+ """Singleton used for logging"""
+
+ _instance = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = object.__new__(cls)
+ cls._instance.init()
+ return cls._instance
+
+ def init(self):
+ cfg = RebuilddConfig()
+ logging.basicConfig(level=logging.DEBUG,
+ format='%(asctime)s %(levelname)s %(message)s',
+ filename=cfg.get('log', 'file'),
+ datefmt=cfg.get('log', 'time_format'),
+ filemode='a')
+
+ @classmethod
+ def info(self, str):
+ """Log a string with info priority"""
+
+ logging.info(str)
+
+ @classmethod
+ def warn(self, str):
+ """Log a string with warn priority"""
+
+ logging.warning(str)
+
+ @classmethod
+ def error(self, str):
+ """Log a string with error priority"""
+
+ logging.error(str)
+
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/RebuilddNetworkClient.py rebuildd-0.4.2/build/lib/rebuildd/RebuilddNetworkClient.py
--- rebuildd-0.4.1/build/lib/rebuildd/RebuilddNetworkClient.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/RebuilddNetworkClient.py 2007-12-03 11:47:42.000000000 +0100
@@ -0,0 +1,212 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+from __future__ import with_statement
+
+from RebuilddConfig import RebuilddConfig
+import threading, socket
+
+__version__ = "$Rev$"
+
+class RebuilddNetworkClient(threading.Thread):
+ """Network client used for each connection"""
+
+ def __init__(self, socket, rebuildd):
+ threading.Thread.__init__(self)
+ self.rebuildd = rebuildd
+ self.socket = socket
+
+ def run(self):
+ """Run client thread"""
+
+ self.socket.settimeout(1)
+ self.socket.send(RebuilddConfig().get('telnet', 'motd') + "\n")
+ prompt = RebuilddConfig().get('telnet', 'prompt') + " "
+ line = ""
+ has_timeout = False
+ while line != "exit" and not self.rebuildd.do_quit.isSet():
+ if not has_timeout:
+ try:
+ self.socket.send(self.exec_cmd(line))
+ self.socket.send(prompt)
+ except Exception:
+ break
+ try:
+ line = ""
+ line = self.socket.recv(512).strip()
+ has_timeout = False
+ except socket.timeout:
+ has_timeout = True
+
+ self.socket.close()
+
+ def exec_cmd(self, cmd):
+ """Execute a command asked by a client"""
+
+ # Return empty if empty
+ if cmd == "":
+ return ""
+
+ # Split words
+ op = cmd.split(' ')
+ cmd_fct = op[0]
+ if not cmd_fct.isalnum():
+ return ""
+ op.remove(op[0])
+ try:
+ return getattr(self, "exec_cmd_" + cmd_fct)(*tuple(op))
+ except Exception, error:
+ return "E: command error: %s\n" % error
+
+ def exec_cmd_help(self, *args):
+ """Show help"""
+
+ help = ""
+ for attr in dir(RebuilddNetworkClient):
+ if attr.startswith("exec_cmd_"):
+ help += "%s -- %s\n" \
+ % (attr[9:].replace("_", " "),
+ getattr(RebuilddNetworkClient, attr).__doc__)
+
+ return help
+
+ def exec_cmd_config(self, *args):
+ """Manipulate configuration file"""
+
+ if len(args) < 1:
+ return "E: usage: config [reload|dump|save]\n"
+
+ if args[0] == "reload":
+ if RebuilddConfig().reload():
+ return "I: config reloaded\n"
+ return "E: config not reloded\n"
+
+ if args[0] == "dump":
+ return RebuilddConfig().dump()
+
+ if args[0] == "save":
+ if RebuilddConfig().save():
+ return "I: config saved\n"
+ return "E: config not saved\n"
+
+ return "E: usage: config [reload|dump|save]\n"
+
+ def exec_cmd_status(self, *args):
+ """Show current jobs status"""
+ return self.rebuildd.dump_jobs()
+
+ def exec_cmd_version(self, *args):
+ """Show version"""
+ return __version__ + "\n"
+
+ def exec_cmd_job(self, *args):
+ """Manipulate jobs"""
+
+ if len(args) > 0 and args[0] == "add":
+ return self.exec_cmd_job_add(*args)
+
+ if len(args) > 0 and args[0] == "cancel":
+ return self.exec_cmd_job_cancel(*args)
+
+ if len(args) > 0 and args[0] == "start":
+ return self.exec_cmd_job_start(*args)
+
+ if len(args) > 0 and args[0] == "reload":
+ return self.exec_cmd_job_reload(*args)
+
+ if len(args) > 0 and args[0] == "status":
+ return self.exec_cmd_job_status(*args)
+
+ return "E: usage: job <command> [args]\n"
+
+ def exec_cmd_job_add(self, *args):
+ """Add job"""
+
+ ret = False
+ if len(args) < 4:
+ return "E: usage: job add <name> <ver> <priority> <dist> [arch] [mailto]\n"
+
+ if len(args) == 5:
+ ret = self.rebuildd.add_job(name=args[1],
+ version=args[2],
+ priority=args[3],
+ dist=args[4])
+
+ if len(args) == 6:
+ ret = self.rebuildd.add_job(name=args[1],
+ version=args[2],
+ priority=args[3],
+ dist=args[4],
+ arch=args[5])
+
+ if len(args) == 7:
+ ret = self.rebuildd.add_job(name=args[1],
+ version=args[2],
+ priority=args[3],
+ dist=args[4],
+ arch=args[5],
+ mailto=args[6])
+
+ if ret:
+ return "I: job added\n"
+ return "E: error adding job\n"
+
+ def exec_cmd_job_cancel(self, *args):
+ """Cancel job"""
+
+ if len(args) < 2:
+ return "E: usage: job cancel <id>\n"
+ if self.rebuildd.cancel_job(int(args[1])):
+ return "I: job canceled\n"
+ return "E: unknown job\n"
+
+ def exec_cmd_job_start(self, *args):
+ """Start jobs"""
+
+ if len(args) == 2:
+ return "I: %s jobs started\n" \
+ % self.rebuildd.start_jobs(int(args[1]))
+ return "I: %s jobs started\n" \
+ % self.rebuildd.start_jobs()
+
+ def exec_cmd_job_reload(self, *args):
+ """Load new jobs"""
+
+ return "I: %s new jobs added\n" % self.rebuildd.get_new_jobs()
+
+ def exec_cmd_job_status(self, *args):
+ """Dump job status"""
+
+ if len(args) < 2 or len(args) > 5:
+ return "E: usage: job status <name> <ver> <dist> [arch]\n"
+ elif len(args) == 2:
+ jobs = self.rebuildd.get_jobs(name=args[1])
+ elif len(args) == 3:
+ jobs = self.rebuildd.get_jobs(name=args[1],
+ version=args[2])
+ elif len(args) == 4:
+ jobs = self.rebuildd.get_jobs(name=args[1],
+ version=args[2],
+ dist=args[3])
+ elif len(args) == 5:
+ jobs = self.rebuildd.get_jobs(name=args[1],
+ version=args[2],
+ dist=args[3],
+ arch=args[4])
+
+ return self.rebuildd.dump_jobs(jobs)
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/RebuilddNetworkServer.py rebuildd-0.4.2/build/lib/rebuildd/RebuilddNetworkServer.py
--- rebuildd-0.4.1/build/lib/rebuildd/RebuilddNetworkServer.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/RebuilddNetworkServer.py 2007-11-19 20:47:35.000000000 +0100
@@ -0,0 +1,50 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+import threading, socket
+from RebuilddConfig import RebuilddConfig
+from RebuilddNetworkClient import RebuilddNetworkClient
+
+class RebuilddNetworkServer(threading.Thread):
+ """Main network server listening for connection"""
+
+ def __init__(self, rebuildd):
+ threading.Thread.__init__(self)
+ self.rebuildd = rebuildd
+
+ def run(self):
+ """Run main network server thread"""
+
+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self.socket.settimeout(1)
+ self.socket.bind((RebuilddConfig().get('telnet', 'ip'),
+ RebuilddConfig().getint('telnet', 'port')))
+ self.socket.listen(2)
+ while not self.rebuildd.do_quit.isSet():
+ try:
+ (client_socket, client_info) = self.socket.accept()
+ if client_socket:
+ interface = RebuilddNetworkClient(client_socket,
+ self.rebuildd)
+ interface.setDaemon(True)
+ interface.start()
+ except socket.timeout:
+ pass
+
+ self.socket.close()
diff -Nru rebuildd-0.4.1/build/lib/rebuildd/Rebuildd.py rebuildd-0.4.2/build/lib/rebuildd/Rebuildd.py
--- rebuildd-0.4.1/build/lib/rebuildd/Rebuildd.py 1970-01-01 01:00:00.000000000 +0100
+++ rebuildd-0.4.2/build/lib/rebuildd/Rebuildd.py 2007-12-03 11:47:42.000000000 +0100
@@ -0,0 +1,379 @@
+# rebuildd - Debian packages rebuild tool
+#
+# (c) 2007 - Julien Danjou <acid@debian.org>
+#
+# This software is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 dated June, 1991.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+#
+
+"""rebuildd - Debian packages rebuild tool"""
+
+from __future__ import with_statement
+
+from Enumeration import Enumeration
+from Distribution import Distribution
+from Dists import Dists
+from RebuilddLog import RebuilddLog
+from RebuilddConfig import RebuilddConfig
+from RebuilddNetworkServer import RebuilddNetworkServer
+from Package import Package
+from Job import Job
+from JobStatus import JobStatus
+from JobStatus import FailedStatus
+import threading, os, time, sys, signal, socket
+import sqlobject
+
+__version__ = "$Rev$"
+
+class Rebuildd(object):
+ jobs = []
+ _instance = None
+
+ def __new__(cls):
+ if cls._instance is None:
+ cls._instance = object.__new__(cls)
+ cls._instance.init()
+ return cls._instance
+
+ def init(self):
+ self.cfg = RebuilddConfig()
+
+ # Init log system
+ RebuilddLog()
+
+ sqlobject.sqlhub.processConnection = \
+ sqlobject.connectionForURI(self.cfg.get('build', 'database_uri'))
+
+ # Create distributions
+ for dist in self.cfg.get('build', 'dists').split(' '):
+ for arch in self.cfg.arch:
+ Dists().add_dist(Distribution(dist, arch))
+
+ self.do_quit = threading.Event()
+ self.jobs_locker = threading.Lock()
+ self.job_finished = threading.Event()
+
+ def daemon(self):
+ RebuilddLog.info("Starting rebuildd %s" % __version__)
+ self.daemonize()
+
+ # Run the network server thread
+ RebuilddLog.info("Launching network server")
+ self.netserv = RebuilddNetworkServer(self)
+ self.netserv.setDaemon(True)
+ self.netserv.start()
+
+ # Run main loop
+ RebuilddLog.info("Running main loop")
+ self.loop()
+
+ # On exit
+ RebuilddLog.info("Cleaning finished and canceled jobs")
+ self.clean_jobs()
+ RebuilddLog.info("Stopping all jobs")
+ self.stop_all_jobs()
+ RebuilddLog.info("Releasing wait-locked jobs")
+ self.release_jobs()
+ self.netserv.join(10)
+ RebuilddLog.info("Exiting rebuildd")
+
+ def daemonize(self):
+ """Do daemon stuff"""
+
+ signal.signal(signal.SIGTERM, self.handle_sigterm)
+ signal.signal(signal.SIGINT, self.handle_sigterm)
+
+ try:
+ os.chdir(self.cfg.get('build', 'work_dir'))
+ except Exception, error:
+ print "E: unable to chdir to work_dir: %s" % error
+ sys.exit(1)
+
+ try:
+ sys.stdout = sys.stderr = file(self.cfg.get('log', 'file'), "a")
+ except Exception, error:
+ print "E: unable to open logfile: %s" % error
+
+ def get_job(self, jobid):
+ for job in self.jobs:
+ if job.id == jobid:
+ return job
+
+ return None
+
+ def get_all_jobs(self, **kwargs):
+ jobs = []
+ jobs.extend(Job.selectBy(**kwargs))
+ return jobs
+
+ def get_new_jobs(self):
+ """Feed jobs list with waiting jobs and lock them"""
+
+ max_new = self.cfg.getint('build', 'max_jobs')
+ count_current = len(self.jobs)
+
+ with self.jobs_locker:
+ if count_current >= max_new:
+ return 0
+
+ jobs = []
+ for dist in Dists().dists:
+ jobs.extend(Job.selectBy(status=JobStatus.WAIT, dist=dist.name, arch=dist.arch)[:max_new])
+
+ count_new = 0
+ for job in jobs:
+ # Look for higher versions ?
+ if self.cfg.getboolean('build', 'build_more_recent'):
+ packages = Package.selectBy(name=job.package.name)
+ candidate_packages = []
+ candidate_packages.extend(packages)
+ candidate_packages.sort(cmp=Package.VersionCompare)
+ candidate_packages.reverse()
+ newjob = None
+
+ # so there are packages with higher version number
+ # try to see if there's a job for us
+ for cpackage in candidate_packages:
+ candidate_jobs = []
+ candidate_jobs.extend(Job.selectBy(package=cpackage, dist=job.dist, arch=job.arch))
+ for cjob in candidate_jobs:
+ if newjob and newjob != cjob and cjob.status == JobStatus.WAIT:
+ cjob.status = JobStatus.GIVEUP
+ elif cjob.status == JobStatus.WAIT:
+ newjob = cjob
+
+ job = newjob
+
+ # We have to check because it might have changed
+ # between our first select and the build_more_recent stuffs
+
+ if not job or job.status != JobStatus.WAIT:
+ continue
+
+ job.status = JobStatus.WAIT_LOCKED
+ job.host = socket.gethostname()
+ self.jobs.append(job)
+ count_new += 1
+ count_current += 1
+
+ if count_current >= max_new:
+ break
+
+ return count_new
+
+ def count_running_jobs(self):
+ """Count running jobs"""
+
+ count = 0
+ with self.jobs_locker:
+ for job in self.jobs:
+ if job.status == JobStatus.BUILDING and job.isAlive():
+ count += 1
+
+ return count
+
+ def start_jobs(self, overrun=0):
+ """Start waiting jobs"""
+
+ running_threads = self.count_running_jobs()
+ max_threads = max(overrun, self.cfg.getint('build', 'max_threads'))
+ jobs_started = 0
+
+ with self.jobs_locker:
+ for job in self.jobs:
+ if running_threads >= max_threads:
+ break
+
+ with job.status_lock:
+ if job.status == JobStatus.WAIT_LOCKED and not job.isAlive():
+ RebuilddLog.info("Starting new thread for job %s" % job.id)
+ job.notify = self.job_finished
+ job.setDaemon(True)
+ job.start()
+ jobs_started += 1
+ running_threads = running_threads + 1
+
+ RebuilddLog.info("Running threads: [ build %s/%s ] [ real %s ]" %
+ (running_threads, max_threads, threading.activeCount()))
+
+ return jobs_started
+
+ def get_jobs(self, name, version=None, dist=None, arch=None):
+ """Dump a job status"""
+
+ if version:
+ pkgs = Package.selectBy(name=name, version=version)
+ else:
+ pkgs = Package.selectBy(name=name)
+
+ if not pkgs.count():
+ return []
+
+ retjobs = []
+ if dist and arch:
+ for pkg in pkgs:
+ retjobs.extend(Job.selectBy(package=pkg, dist=dist, arch=arch))
+ elif dist:
+ for pkg in pkgs:
+ retjobs.extend(Job.selectBy(package=pkg, dist=dist))
+ elif arch:
+ for pkg in pkgs:
+ retjobs.extend(Job.selectBy(package=pkg, arch=arch))
+ else:
+ for pkg in pkgs:
+ retjobs.extend(Job.selectBy(package=pkg))
+
+ return retjobs
+
+ def dump_jobs(self, joblist=None):
+ """Dump all jobs status"""
+
+ ret = ""
+
+ if not joblist:
+ joblist = self.jobs
+
+ for job in joblist:
+ ret = "%s%s\n" % (ret, str(job))
+
+ return ret
+
+ def cancel_job(self, jobid):
+ """Cancel a job"""
+
+ with self.jobs_locker:
+ job = self.get_job(jobid)
+ if job != None:
+ if job.isAlive():
+ job.do_quit.set()
+ job.join()
+ with job.status_lock:
+ job.status = JobStatus.CANCELED
+ self.jobs.remove(job)
+ RebuilddLog.info("Canceled job %s for %s_%s on %s/%s for %s" \
+ % (job.id, job.package.name, job.package.version,
+ job.dist, job.arch, job.mailto))
+ return True
+
+ return False
+
+ def stop_all_jobs(self):
+ """Stop all running jobs"""
+
+ with self.jobs_locker:
+ for job in self.jobs:
+ if job.status == JobStatus.BUILDING and job.isAlive():
+ job.do_quit.set()
+ RebuilddLog.info("Sending stop to job %s" % job.id)
+ for job in self.jobs:
+ if job.isAlive():
+ RebuilddLog.info("Waiting for job %s to terminate" % job.id)
+ job.join(60)
+
+ return True
+
+ def add_job(self, name, version, priority, dist, mailto=None, arch=None):
+ """Add a job"""
+
+ if not arch:
+ arch = self.cfg.arch[0]
+
+ if not Dists().get_dist(dist, arch):
+ return False
+
+ pkgs = Package.selectBy(name=name, version=version)
+ if pkgs.count():
+ # If several packages exists, just take the first
+ pkg = pkgs[0]
+ else:
+ # Maybe we found no packages, so create a brand new one!
+ pkg = Package(name=name, version=version, priority=priority)
+
+ jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
+ if jobs_count:
+ RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
+ % (pkg.name, pkg.version, dist, arch))
+ return False
+
+ job = Job(package=pkg, dist=dist, arch=arch)
+ job.status = JobStatus.WAIT
+ job.arch = arch
+ job.mailto = mailto
+
+ RebuilddLog.info("Added job for %s_%s on %s/%s for %s" \
+ % (name, version, dist, arch, mailto))
+ return True
+
+ def clean_jobs(self):
+ """Clean finished or canceled jobs"""
+
+ with self.jobs_locker:
+ for job in self.jobs:
+ if job.status == JobStatus.BUILD_OK \
+ or job.status in FailedStatus \
+ or job.status == JobStatus.CANCELED:
+ self.jobs.remove(job)
+
+ return True
+
+ def release_jobs(self):
+ """Release jobs"""
+
+ with self.jobs_locker:
+ for job in self.jobs:
+ with job.status_lock:
+ if job.status == JobStatus.WAIT_LOCKED:
+ job.status = JobStatus.WAIT
+ job.host = ""
+
+ return True
+
+ def fix_jobs(self, print_result=True):
+ """If rebuildd crashed, reset jobs to a valid state"""
+
+ jobs = []
+ jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.WAIT_LOCKED))
+ jobs.extend(Job.selectBy(host=socket.gethostname(), status=JobStatus.BUILDING))
+
+ for job in jobs:
+ if print_result:
+ print "I: Fixing job %s (was %s)" % (job.id, JobStatus.whatis(job.status))
+ job.host = None
+ job.status = JobStatus.WAIT
+ job.build_start = None
+ job.build_end = None
+
+ return True
+
+ def handle_sigterm(self, signum, stack):
+ RebuilddLog.info("Receiving transmission... it's a signal %s capt'ain! EVERYONE OUT!" % signum)
+ self.do_quit.set()
+
+ def loop(self):
+ """Rebuildd main loop"""
+
+ counter = self.cfg.getint('build', 'check_every')
+ while not self.do_quit.isSet():
+ if counter == self.cfg.getint('build', 'check_every') \
+ or self.job_finished.isSet():
+ self.get_new_jobs()
+ # Start jobs
+ self.start_jobs()
+ # Clean finished jobs
+ self.clean_jobs()
+ counter = 0
+ self.job_finished.clear()
+ self.do_quit.wait(1)
+ counter += 1
+
+
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/__init__.py rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/__init__.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/__init__.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/__init__.py 2012-07-24 10:59:36.000000000 +0200
@@ -19,8 +19,6 @@
"""rebuildd - Debian packages rebuild tool"""
__author__ = "Julien Danjou <acid@debian.org>"
-__cvsid__ = "$Id$"
-__version__ = "$Rev$"[6:-2]
__all__ = [
"Distribution",
"Enumeration",
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/Job.py rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/Job.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/Job.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/Job.py 2012-07-24 10:59:36.000000000 +0200
@@ -20,6 +20,7 @@
import threading, subprocess, smtplib, time, os, signal, socket, select
import sqlobject
+from subprocess import Popen,PIPE
from email.Message import Message
from Dists import Dists
from JobStatus import JobStatus
@@ -52,8 +53,6 @@
threading.Thread.__init__(self)
sqlobject.SQLObject.__init__(self, *args, **kwargs)
- if self.log is None:
- log = Log(job=self)
self.do_quit = threading.Event()
self.status_lock = threading.Lock()
@@ -201,7 +200,8 @@
return False
# Store in database
- self.log.text = log
+ if self.log:
+ self.log.text = log
with self.status_lock:
if self.status != JobStatus.BUILD_OK and \
@@ -243,16 +243,15 @@
smtp = smtplib.SMTP()
smtp.connect(RebuilddConfig().get('mail', 'smtp_host'),
RebuilddConfig().get('mail', 'smtp_port'))
- if self.mailto:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- self.mailto,
- msg.as_string())
- else:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- RebuilddConfig().get('mail', 'mailto'),
- msg.as_string())
+ smtp.sendmail(RebuilddConfig().get('mail', 'from'),
+ [m.strip() for m in msg['To'].split(",")],
+ msg.as_string())
except Exception, error:
- RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
+ try:
+ process = Popen("sendmail", shell=True, stdin=PIPE)
+ process.communicate(input=msg.as_string())
+ except:
+ RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
return True
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/RebuilddConfig.py rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/RebuilddConfig.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/RebuilddConfig.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/RebuilddConfig.py 2012-07-24 10:59:36.000000000 +0200
@@ -53,6 +53,7 @@
self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db')
self.set('build', 'build_more_recent', '1')
self.set('build', 'more_archs', 'any')
+ self.set('build', 'no_system_arch', '0')
self.set('mail', 'from', 'rebuildd@localhost')
self.set('mail', 'mailto', 'rebuildd@localhost')
@@ -74,19 +75,20 @@
self.set('http', 'logfile', '/var/log/rebuildd/httpd.log')
self.set('log', 'file', "/var/log/rebuildd/rebuildd.log")
- self.set('log', 'time_format', "%d-%m-%Y %H:%M:%S")
+ self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S")
self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs")
self.set('log', 'mail_failed', '1')
self.set('log', 'mail_successful', '0')
- self.arch = []
- parch = os.popen("dpkg --print-architecture")
- self.arch.append(parch.readline().strip())
- parch.close()
-
if not dontparse:
self.reload()
+ self.arch = []
+ if self.getint('build', 'no_system_arch') == 0:
+ parch = os.popen("dpkg --print-architecture")
+ self.arch.append(parch.readline().strip())
+ parch.close()
+
for a in self.get('build', 'more_archs').split(' '):
self.arch.append(a)
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkClient.py rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkClient.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkClient.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/RebuilddNetworkClient.py 2012-07-24 10:59:36.000000000 +0200
@@ -135,6 +135,9 @@
if len(args) > 0 and args[0] == "status":
return self.exec_cmd_job_status(*args)
+ if len(args) > 0 and args[0] == "requeue":
+ return self.exec_cmd_job_requeue(*args)
+
return "E: usage: job <command> [args]\n"
def exec_cmd_job_add(self, *args):
@@ -169,12 +172,21 @@
return "I: job added\n"
return "E: error adding job\n"
+ def exec_cmd_job_requeue(self, *args):
+ """Requeue job"""
+
+ if len(args) != 2:
+ return "E: usage: job requeue <job_id>\n"
+
+ if self.rebuildd.requeue_job(job_id=int(args[1])):
+ return "I: job requeued\n"
+ return "E: error requeuing the job\n"
def exec_cmd_job_deps(self, *args):
"""Add dependency"""
ret = False
- if len(args) < 2:
+ if len(args) < 3:
return "E: usage: job deps <job_id> <dependency_job_id> [dependency_job_id] [...]\n"
ret = self.rebuildd.add_deps(job_id=args[1],
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/Rebuildd.py rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/Rebuildd.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.6/rebuildd/Rebuildd.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.6/rebuildd/Rebuildd.py 2012-07-24 10:59:36.000000000 +0200
@@ -293,7 +293,7 @@
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
- RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \
% (name, version, dist, arch))
return False
@@ -307,7 +307,7 @@
jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
if jobs_count:
- RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
@@ -322,6 +322,20 @@
% (name, version, dist, arch, mailto))
return True
+ def requeue_job(self, job_id):
+ """Requeue a failed job"""
+
+ if Job.selectBy(id=job_id).count() == 0:
+ RebuilddLog.error("There is no job related to %s that is in the job list" % job_id)
+ return False
+ job = Job.selectBy(id=job_id)[0]
+
+ if job.status in FailedStatus:
+ job.status = JobStatus.WAIT
+ job.host = ""
+
+ return True
+
def add_deps(self, job_id, dependency_ids):
if Job.selectBy(id=job_id).count() == 0:
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/__init__.py rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/__init__.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/__init__.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/__init__.py 2012-07-24 10:59:36.000000000 +0200
@@ -19,8 +19,6 @@
"""rebuildd - Debian packages rebuild tool"""
__author__ = "Julien Danjou <acid@debian.org>"
-__cvsid__ = "$Id$"
-__version__ = "$Rev$"[6:-2]
__all__ = [
"Distribution",
"Enumeration",
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/Job.py rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/Job.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/Job.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/Job.py 2012-07-24 10:59:36.000000000 +0200
@@ -20,6 +20,7 @@
import threading, subprocess, smtplib, time, os, signal, socket, select
import sqlobject
+from subprocess import Popen,PIPE
from email.Message import Message
from Dists import Dists
from JobStatus import JobStatus
@@ -52,8 +53,6 @@
threading.Thread.__init__(self)
sqlobject.SQLObject.__init__(self, *args, **kwargs)
- if self.log is None:
- log = Log(job=self)
self.do_quit = threading.Event()
self.status_lock = threading.Lock()
@@ -201,7 +200,8 @@
return False
# Store in database
- self.log.text = log
+ if self.log:
+ self.log.text = log
with self.status_lock:
if self.status != JobStatus.BUILD_OK and \
@@ -243,16 +243,15 @@
smtp = smtplib.SMTP()
smtp.connect(RebuilddConfig().get('mail', 'smtp_host'),
RebuilddConfig().get('mail', 'smtp_port'))
- if self.mailto:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- self.mailto,
- msg.as_string())
- else:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- RebuilddConfig().get('mail', 'mailto'),
- msg.as_string())
+ smtp.sendmail(RebuilddConfig().get('mail', 'from'),
+ [m.strip() for m in msg['To'].split(",")],
+ msg.as_string())
except Exception, error:
- RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
+ try:
+ process = Popen("sendmail", shell=True, stdin=PIPE)
+ process.communicate(input=msg.as_string())
+ except:
+ RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
return True
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/RebuilddConfig.py rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/RebuilddConfig.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/RebuilddConfig.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/RebuilddConfig.py 2012-07-24 10:59:36.000000000 +0200
@@ -53,6 +53,7 @@
self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db')
self.set('build', 'build_more_recent', '1')
self.set('build', 'more_archs', 'any')
+ self.set('build', 'no_system_arch', '0')
self.set('mail', 'from', 'rebuildd@localhost')
self.set('mail', 'mailto', 'rebuildd@localhost')
@@ -74,19 +75,20 @@
self.set('http', 'logfile', '/var/log/rebuildd/httpd.log')
self.set('log', 'file', "/var/log/rebuildd/rebuildd.log")
- self.set('log', 'time_format', "%d-%m-%Y %H:%M:%S")
+ self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S")
self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs")
self.set('log', 'mail_failed', '1')
self.set('log', 'mail_successful', '0')
- self.arch = []
- parch = os.popen("dpkg --print-architecture")
- self.arch.append(parch.readline().strip())
- parch.close()
-
if not dontparse:
self.reload()
+ self.arch = []
+ if self.getint('build', 'no_system_arch') == 0:
+ parch = os.popen("dpkg --print-architecture")
+ self.arch.append(parch.readline().strip())
+ parch.close()
+
for a in self.get('build', 'more_archs').split(' '):
self.arch.append(a)
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkClient.py rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkClient.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkClient.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/RebuilddNetworkClient.py 2012-07-24 10:59:36.000000000 +0200
@@ -135,6 +135,9 @@
if len(args) > 0 and args[0] == "status":
return self.exec_cmd_job_status(*args)
+ if len(args) > 0 and args[0] == "requeue":
+ return self.exec_cmd_job_requeue(*args)
+
return "E: usage: job <command> [args]\n"
def exec_cmd_job_add(self, *args):
@@ -169,12 +172,21 @@
return "I: job added\n"
return "E: error adding job\n"
+ def exec_cmd_job_requeue(self, *args):
+ """Requeue job"""
+
+ if len(args) != 2:
+ return "E: usage: job requeue <job_id>\n"
+
+ if self.rebuildd.requeue_job(job_id=int(args[1])):
+ return "I: job requeued\n"
+ return "E: error requeuing the job\n"
def exec_cmd_job_deps(self, *args):
"""Add dependency"""
ret = False
- if len(args) < 2:
+ if len(args) < 3:
return "E: usage: job deps <job_id> <dependency_job_id> [dependency_job_id] [...]\n"
ret = self.rebuildd.add_deps(job_id=args[1],
diff -Nru rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/Rebuildd.py rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/Rebuildd.py
--- rebuildd-0.4.1/build/lib.linux-x86_64-2.7/rebuildd/Rebuildd.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/build/lib.linux-x86_64-2.7/rebuildd/Rebuildd.py 2012-07-24 10:59:36.000000000 +0200
@@ -293,7 +293,7 @@
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
- RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \
% (name, version, dist, arch))
return False
@@ -307,7 +307,7 @@
jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
if jobs_count:
- RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
@@ -322,6 +322,20 @@
% (name, version, dist, arch, mailto))
return True
+ def requeue_job(self, job_id):
+ """Requeue a failed job"""
+
+ if Job.selectBy(id=job_id).count() == 0:
+ RebuilddLog.error("There is no job related to %s that is in the job list" % job_id)
+ return False
+ job = Job.selectBy(id=job_id)[0]
+
+ if job.status in FailedStatus:
+ job.status = JobStatus.WAIT
+ job.host = ""
+
+ return True
+
def add_deps(self, job_id, dependency_ids):
if Job.selectBy(id=job_id).count() == 0:
diff -Nru rebuildd-0.4.1/debian/changelog rebuildd-0.4.2/debian/changelog
--- rebuildd-0.4.1/debian/changelog 2012-03-09 11:48:42.000000000 +0100
+++ rebuildd-0.4.2/debian/changelog 2012-07-24 11:12:42.000000000 +0200
@@ -1,3 +1,23 @@
+rebuildd (0.4.2) unstable; urgency=low
+
+ [ Gauvain Pocentek ]
+ * Remove unused code (Closes: #671635)
+ * Support multiple recipients
+ * Improve some error messages
+ * Add a requeue command
+ * Fix the number of arguments for jobs deps
+ * Allow to use sendmail
+ * Change the default date format
+ * Add no_system_arch option
+ * Reload before adding arch
+
+ [ Julien Danjou ]
+ * Update default dists to symbolic names
+ * Fix PBUILDER_OTHER_OPTIONS[] examples in rebuildd.default
+ (Closes: #682422)
+
+ -- Julien Danjou <acid@debian.org> Thu, 24 May 2012 16:50:03 +0200
+
rebuildd (0.4.1) unstable; urgency=low
[ Daniel Dehennin ]
diff -Nru rebuildd-0.4.1/debian/rebuildd.default rebuildd-0.4.2/debian/rebuildd.default
--- rebuildd-0.4.1/debian/rebuildd.default 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/debian/rebuildd.default 2012-07-24 11:09:24.000000000 +0200
@@ -21,12 +21,14 @@
# PBUILDER_MIRROR=http://ftp.debian.org/debian
# Pass other options to pbuilder
-# PBUILDER_OTHER_OPTIONS[0]="--components=main contrib"
-# PBUILDER_OTHER_OPTIONS[1]="--othermirror=deb http://my.apt.repo sid main"
+# PBUILDER_OTHER_OPTIONS[0]="--components"
+# PBUILDER_OTHER_OPTIONS[1]="main contrib"
+# PBUILDER_OTHER_OPTIONS[2]="--othermirror"
+# PBUILDER_OTHER_OPTIONS[3]="deb http://my.apt.repo sid main"
# Distributions to generate and manage
ARCHS="$(dpkg --print-architecture)"
-DISTS="etch lenny sid"
+DISTS="stable testing unstable"
# Set to 1 to enable pbuilder/cowbuilder update in cron
ENABLE_BUILDER_MAINT=0
diff -Nru rebuildd-0.4.1/debian/rebuildd-job.manpage.xml rebuildd-0.4.2/debian/rebuildd-job.manpage.xml
--- rebuildd-0.4.1/debian/rebuildd-job.manpage.xml 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/debian/rebuildd-job.manpage.xml 2012-07-24 10:59:36.000000000 +0200
@@ -87,6 +87,15 @@
</listitem>
</varlistentry>
<varlistentry>
+ <term><option>requeue [jobid]</option>
+ </term>
+ <listitem>
+ <para>Reschedule a job.
+ The job id has to be provided as argument. The job must be in
+ an error state (previous build failed).</para>
+ </listitem>
+ </varlistentry>
+ <varlistentry>
<term><option>delete [jobid]</option>
</term>
<listitem>
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Distribution.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Distribution.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Dists.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Dists.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Enumeration.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Enumeration.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd/__init__.py rebuildd-0.4.2/rebuildd/__init__.py
--- rebuildd-0.4.1/rebuildd/__init__.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/rebuildd/__init__.py 2012-07-24 10:59:36.000000000 +0200
@@ -19,8 +19,6 @@
"""rebuildd - Debian packages rebuild tool"""
__author__ = "Julien Danjou <acid@debian.org>"
-__cvsid__ = "$Id$"
-__version__ = "$Rev$"[6:-2]
__all__ = [
"Distribution",
"Enumeration",
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/__init__.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/__init__.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd/Job.py rebuildd-0.4.2/rebuildd/Job.py
--- rebuildd-0.4.1/rebuildd/Job.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/rebuildd/Job.py 2012-07-24 10:59:36.000000000 +0200
@@ -20,6 +20,7 @@
import threading, subprocess, smtplib, time, os, signal, socket, select
import sqlobject
+from subprocess import Popen,PIPE
from email.Message import Message
from Dists import Dists
from JobStatus import JobStatus
@@ -52,8 +53,6 @@
threading.Thread.__init__(self)
sqlobject.SQLObject.__init__(self, *args, **kwargs)
- if self.log is None:
- log = Log(job=self)
self.do_quit = threading.Event()
self.status_lock = threading.Lock()
@@ -201,7 +200,8 @@
return False
# Store in database
- self.log.text = log
+ if self.log:
+ self.log.text = log
with self.status_lock:
if self.status != JobStatus.BUILD_OK and \
@@ -243,16 +243,15 @@
smtp = smtplib.SMTP()
smtp.connect(RebuilddConfig().get('mail', 'smtp_host'),
RebuilddConfig().get('mail', 'smtp_port'))
- if self.mailto:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- self.mailto,
- msg.as_string())
- else:
- smtp.sendmail(RebuilddConfig().get('mail', 'from'),
- RebuilddConfig().get('mail', 'mailto'),
- msg.as_string())
+ smtp.sendmail(RebuilddConfig().get('mail', 'from'),
+ [m.strip() for m in msg['To'].split(",")],
+ msg.as_string())
except Exception, error:
- RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
+ try:
+ process = Popen("sendmail", shell=True, stdin=PIPE)
+ process.communicate(input=msg.as_string())
+ except:
+ RebuilddLog.error("Unable to send build log mail for job %d: %s" % (self.id, error))
return True
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Job.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Job.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/JobStatus.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/JobStatus.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Package.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Package.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd/RebuilddConfig.py rebuildd-0.4.2/rebuildd/RebuilddConfig.py
--- rebuildd-0.4.1/rebuildd/RebuilddConfig.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/rebuildd/RebuilddConfig.py 2012-07-24 10:59:36.000000000 +0200
@@ -53,6 +53,7 @@
self.set('build', 'database_uri', 'sqlite:///var/lib/rebuildd/rebuildd.db')
self.set('build', 'build_more_recent', '1')
self.set('build', 'more_archs', 'any')
+ self.set('build', 'no_system_arch', '0')
self.set('mail', 'from', 'rebuildd@localhost')
self.set('mail', 'mailto', 'rebuildd@localhost')
@@ -74,19 +75,20 @@
self.set('http', 'logfile', '/var/log/rebuildd/httpd.log')
self.set('log', 'file', "/var/log/rebuildd/rebuildd.log")
- self.set('log', 'time_format', "%d-%m-%Y %H:%M:%S")
+ self.set('log', 'time_format', "%Y-%m-%d %H:%M:%S")
self.set('log', 'logs_dir', "/var/log/rebuildd/build_logs")
self.set('log', 'mail_failed', '1')
self.set('log', 'mail_successful', '0')
- self.arch = []
- parch = os.popen("dpkg --print-architecture")
- self.arch.append(parch.readline().strip())
- parch.close()
-
if not dontparse:
self.reload()
+ self.arch = []
+ if self.getint('build', 'no_system_arch') == 0:
+ parch = os.popen("dpkg --print-architecture")
+ self.arch.append(parch.readline().strip())
+ parch.close()
+
for a in self.get('build', 'more_archs').split(' '):
self.arch.append(a)
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/RebuilddConfig.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/RebuilddConfig.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/RebuilddLog.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/RebuilddLog.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd/RebuilddNetworkClient.py rebuildd-0.4.2/rebuildd/RebuilddNetworkClient.py
--- rebuildd-0.4.1/rebuildd/RebuilddNetworkClient.py 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/rebuildd/RebuilddNetworkClient.py 2012-07-24 10:59:36.000000000 +0200
@@ -135,6 +135,9 @@
if len(args) > 0 and args[0] == "status":
return self.exec_cmd_job_status(*args)
+ if len(args) > 0 and args[0] == "requeue":
+ return self.exec_cmd_job_requeue(*args)
+
return "E: usage: job <command> [args]\n"
def exec_cmd_job_add(self, *args):
@@ -169,12 +172,21 @@
return "I: job added\n"
return "E: error adding job\n"
+ def exec_cmd_job_requeue(self, *args):
+ """Requeue job"""
+
+ if len(args) != 2:
+ return "E: usage: job requeue <job_id>\n"
+
+ if self.rebuildd.requeue_job(job_id=int(args[1])):
+ return "I: job requeued\n"
+ return "E: error requeuing the job\n"
def exec_cmd_job_deps(self, *args):
"""Add dependency"""
ret = False
- if len(args) < 2:
+ if len(args) < 3:
return "E: usage: job deps <job_id> <dependency_job_id> [dependency_job_id] [...]\n"
ret = self.rebuildd.add_deps(job_id=args[1],
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/RebuilddNetworkClient.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/RebuilddNetworkClient.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/RebuilddNetworkServer.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/RebuilddNetworkServer.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd/Rebuildd.py rebuildd-0.4.2/rebuildd/Rebuildd.py
--- rebuildd-0.4.1/rebuildd/Rebuildd.py 2012-02-13 12:11:47.000000000 +0100
+++ rebuildd-0.4.2/rebuildd/Rebuildd.py 2012-07-24 10:59:36.000000000 +0200
@@ -293,7 +293,7 @@
arch = self.cfg.arch[0]
if not Dists().get_dist(dist, arch):
- RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Couldn't find dist/arch in the config file for %s_%s on %s/%s, not adding it" \
% (name, version, dist, arch))
return False
@@ -307,7 +307,7 @@
jobs_count = Job.selectBy(package=pkg, dist=dist, arch=arch, mailto=mailto, status=JobStatus.WAIT).count()
if jobs_count:
- RebuilddLog.error("Job already existing for %s_%s on %s/%s, don't adding it" \
+ RebuilddLog.error("Job already existing for %s_%s on %s/%s, not adding it" \
% (pkg.name, pkg.version, dist, arch))
return False
@@ -322,6 +322,20 @@
% (name, version, dist, arch, mailto))
return True
+ def requeue_job(self, job_id):
+ """Requeue a failed job"""
+
+ if Job.selectBy(id=job_id).count() == 0:
+ RebuilddLog.error("There is no job related to %s that is in the job list" % job_id)
+ return False
+ job = Job.selectBy(id=job_id)[0]
+
+ if job.status in FailedStatus:
+ job.status = JobStatus.WAIT
+ job.host = ""
+
+ return True
+
def add_deps(self, job_id, dependency_ids):
if Job.selectBy(id=job_id).count() == 0:
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/rebuildd/Rebuildd.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/rebuildd/Rebuildd.pyc differ
diff -Nru rebuildd-0.4.1/rebuildd-job rebuildd-0.4.2/rebuildd-job
--- rebuildd-0.4.1/rebuildd-job 2011-10-05 16:05:33.000000000 +0200
+++ rebuildd-0.4.2/rebuildd-job 2012-07-24 10:59:36.000000000 +0200
@@ -32,6 +32,7 @@
print " add - add jobs, reading from stdin"
print " add-deps - add build-depends, reading from stdin"
print " add-quinn-diff <dist> - add package reading quinn-diff input from stdin"
+ print " requeue <jobid> - requeue job for rebuild"
print " delete <jobid> - delete job"
print " list-deps - list the whole depends table"
print " list <criteria>=<value> - list jobs matching criteria"
@@ -84,6 +85,11 @@
priority = line.split('/')[1].split('_')[1].split('.dsc')[1].split(':')[0][2:]
Rebuildd().add_job(name, version, priority, dist)
+def requeue_job(jobid):
+ if not Rebuildd().requeue_job(jobid):
+ print "E: rebuild not accepted"
+ sys.exit(1)
+
def list():
if len(sys.argv) == 3:
try:
@@ -214,5 +220,7 @@
stats()
if sys.argv[1] == "delete" and len(sys.argv) > 2:
delete(int(sys.argv[2]))
+ if sys.argv[1] == "requeue" and len(sys.argv) == 3:
+ requeue_job(int(sys.argv[2]))
else:
usage()
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/tests/RebuilddTestSetup.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/tests/RebuilddTestSetup.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/tests/TestDistribution.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/tests/TestDistribution.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/tests/TestJob.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/tests/TestJob.pyc differ
Binary files /tmp/Xvndfh3Gc9/rebuildd-0.4.1/tests/TestRebuildd.pyc and /tmp/DPQUc59v_l/rebuildd-0.4.2/tests/TestRebuildd.pyc differ
unblock rebuildd/0.4.2
-- System Information:
Debian Release: 7.0
APT prefers unstable
APT policy: (500, 'unstable'), (1, 'experimental')
Architecture: amd64 (x86_64)
Foreign Architectures: i386
Kernel: Linux 3.7-trunk-amd64 (SMP w/4 CPU cores)
Locale: LANG=en_US.UTF-8, LC_CTYPE=en_US.UTF-8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Reply to: