[snapshot/master] script to import all mirrorruns found in a dump-git-repository
---
misc/dump-tools/import-new-dumps-from-git | 99 +++++++++++++++++++++++++++++
1 files changed, 99 insertions(+), 0 deletions(-)
create mode 100755 misc/dump-tools/import-new-dumps-from-git
diff --git a/misc/dump-tools/import-new-dumps-from-git b/misc/dump-tools/import-new-dumps-from-git
new file mode 100755
index 0000000..b977985
--- /dev/null
+++ b/misc/dump-tools/import-new-dumps-from-git
@@ -0,0 +1,99 @@
+#!/usr/bin/python
+
+# Copyright (c) 2010 Peter Palfrader
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+# Imports all mirrorruns found in a git repository for dumps into the database.
+# Just imports metadata, does not get any actualy content (i.e. does not
+# populate the farm).
+
+import sys
+import yaml
+import optparse
+import os
+import tempfile
+import subprocess
+thisscriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
+sys.path.append(os.path.join(thisscriptdir, '../../lib'))
+from dbhelper import DBHelper
+
+parser = optparse.OptionParser()
+parser.set_usage("%prog --config=<conffile>")
+parser.add_option("-c", "--config", dest="conffile", metavar="CONFFILE",
+ help="Config file location.")
+parser.add_option("-v", "--verbose", action="store_true",
+ help="Be verbose.")
+parser.add_option("-s", "--snapshot", action="store", metavar="PATH",
+ help="Path to the snapshot script.")
+parser.add_option("-e", "--extracter", action="store", metavar="PATH",
+ help="Path to the extract-dumps script.")
+parser.add_option("-b", "--backing", dest="backing_git", metavar="GITDIR",
+ help="Location of backing git working copy.")
+
+(options, args) = parser.parse_args()
+if options.conffile is None:
+ parser.print_help()
+ sys.exit(1)
+
+if options.snapshot is None:
+ options.snapshot = os.path.join(thisscriptdir, '../../snapshot')
+if options.extracter is None:
+ options.extracter = os.path.join(thisscriptdir, 'extract-dumps')
+
+config = yaml.load(open(options.conffile).read())
+db = DBHelper(config['db']['connectstring'])
+
+if not os.path.exists(options.snapshot) or not os.access(options.snapshot, os.X_OK):
+ sys.stderr.write("%s does not exist or is not exectuable\n"%(options.snapshot))
+ sys.exit(1)
+if not os.path.exists(options.extracter) or not os.access(options.extracter, os.X_OK):
+ sys.stderr.write("%s does not exist or is not exectuable\n"%(options.extracter))
+ sys.exit(1)
+
+options.extracter = os.path.abspath(options.extracter)
+extractcall = [options.extracter]
+if options.backing_git is not None: extractcall += ['--backing', options.backing_git]
+
+pending_import = []
+p = subprocess.Popen(extractcall+['list'], stdout=subprocess.PIPE)
+for line in p.stdout:
+ line = line.rstrip()
+ uuid, objectspec = line.split(None, 1)
+ row = db.query_one('SELECT mirrorrun_id FROM mirrorrun WHERE mirrorrun_uuid=%(uuid)s', {'uuid': uuid})
+ if row is None:
+ pending_import.append(objectspec)
+db.close()
+
+for objectspec in pending_import:
+ if options.verbose: print "Doing %s."%(objectspec)
+ t = tempfile.NamedTemporaryFile(suffix='.tmp', prefix='git-dump-import-')
+ p = subprocess.Popen(extractcall+['get', objectspec], stdout=t)
+ t.flush()
+
+ c = [options.snapshot, '-c', options.conffile, '-p', t.name, '--quick', 'import-dump']
+ if options.verbose: c.append('--verbose')
+ subprocess.check_call(c)
+
+ t.close()
+
+# vim:set et:
+# vim:set ts=4:
+# vim:set shiftwidth=4:
--
1.7.2.3
Reply to: