#22 tools to dump/restore hosts
Merged 6 years ago by mikem. Opened 6 years ago by mikem.
mikem/koji-tools dump-hosts  into  master

file modified
+4
@@ -51,3 +51,7 @@ 

  * `koji-tag-overlap` - Show package overlaps for a set of tags

  

  * `kojitop` - Show the tasks each builder is working on

+ 

+ * `koji-dump-hosts` - Write current host data to a file

+ 

+ * `koji-restore-hosts` - Restore host data from a file

@@ -0,0 +1,68 @@ 

+ #!/usr/bin/python

+ 

+ import json

+ import optparse

+ import os

+ import sys

+ 

+ import koji

+ from koji_cli.lib import _

+ 

+ 

+ def main():

+     global koji

+     global session

+     parser = optparse.OptionParser(usage='%prog [options]')

+     parser.add_option('-p', '--profile', default='koji', help='pick a profile')

+     parser.add_option('-o', '--outfile', help='write data to file')

+     parser.add_option("--arch", action="append", default=[], help=_("Specify an architecture"))

+     parser.add_option("--channel", help=_("Specify a channel"))

+     parser.add_option("--enabled", action="store_true", help=_("Limit to enabled hosts"))

+     parser.add_option("--not-enabled", action="store_false", dest="enabled", help=_("Limit to not enabled hosts"))

+ 

+     opts, args = parser.parse_args()

+ 

+     if args:

+         parser.error('Unexpected argument')

+ 

+     koji = koji.get_profile_module(opts.profile)

+ 

+     for name in ('cert', 'serverca'):

+         value = os.path.expanduser(getattr(koji.config, name))

+         setattr(koji.config, name, value)

+ 

+     session_opts = koji.grab_session_options(koji.config)

+     session = koji.ClientSession(koji.config.server, session_opts)

+ 

+     data = get_host_data(opts)

+     if opts.outfile:

+         with open(opts.outfile, 'w') as fp:

+             json.dump(data, fp, indent=4)

+     else:

+         json.dump(data, sys.stdout, indent=4)

+ 

+ 

+ def get_host_data(options):

+     opts = {}

+     if options.arch:

+         opts['arches'] = options.arch

+     if options.channel:

+         channel = session.getChannel(options.channel, strict=True)

+         opts['channelID'] = channel['id']

+     if options.enabled is not None:

+         opts['enabled'] = options.enabled

+ 

+     hosts = session.listHosts(**opts)

+ 

+     # also fetch channels

+     session.multicall = True

+     for host in hosts:

+         session.listChannels(hostID=host['id'])

+     for host, [channels] in zip(hosts, session.multiCall(strict=True)):

+         host['channels'] = channels

+ 

+     return hosts

+ 

+ 

+ if __name__ == '__main__':

+     main()

@@ -0,0 +1,147 @@ 

+ #!/usr/bin/python

+ 

+ import json

+ import optparse

+ import os

+ import sys

+ 

+ import koji

+ from koji.util import dslice

+ from koji_cli.lib import activate_session

+ 

+ 

+ def main():

+     global koji

+     global session

+     parser = optparse.OptionParser(usage='%prog [options]')

+     parser.add_option('-p', '--profile', default='koji', help='pick a profile')

+     parser.add_option('-i', '--infile', help='read data from file')

+     parser.add_option('-n', '--test', action='store_true', default=False,

+                       help='test mode')

+     opts, args = parser.parse_args()

+ 

+     if args:

+         parser.error('Unexpected argument')

+ 

+     koji = koji.get_profile_module(opts.profile)

+ 

+     for name in ('cert', 'serverca'):

+         value = os.path.expanduser(getattr(koji.config, name))

+         setattr(koji.config, name, value)

+ 

+     session_opts = koji.grab_session_options(koji.config)

+     session = koji.ClientSession(koji.config.server, session_opts)

+ 

+     if not opts.test:

+         activate_session(session, koji.config)

+ 

+     data = read_host_data(opts)

+     print("Read %i host entries" % len(data))

+ 

+     changes = compare_hosts(data)

+ 

+     if opts.test:

+         print_changes(changes)

+     else:

+         do_changes(changes)

+ 

+ 

+ def read_host_data(opts):

+     if opts.infile:

+         with open(opts.infile, 'r') as fp:

+             return json.load(fp)

+     else:

+         return json.load(sys.stdin)

+ 

+ 

+ def compare_hosts(data):

+     old_data = get_host_data()

+     changes = []

+     o_idx = dict([[h['name'], h] for h in old_data])

+     n_idx = dict([[h['name'], h] for h in data])

+ 

+     changes_new = get_new_hosts(o_idx, n_idx)

+     changes_update = get_host_updates(o_idx, n_idx)

+ 

+     return changes_new + changes_update

+ 

+ 

+ def get_new_hosts(o_idx, n_idx):

+     added = set(n_idx) - set(o_idx)

+     changes = []

+     for name in added:

+         host = n_idx[name]

+         # fields we care about: arches, capacity, description, comment, enabled

+         #  channels

+         archlist = host['arches'].split()

+         changes.append(['addHost', [name, archlist], {}])

+         # unfortunately, addHost cannot set all the fields we need to

+         edits = dslice(host, ['capacity', 'description', 'comment'])

+         changes.append(['editHost', [name], edits])

+         for channel in host['channels']:

+             changes.append(['addHostToChannel', [name, channel['name']], {}])

+             # TODO: option to create channel

+         # new host entry will be enabled by default

+         if not host['enabled']:

+             changes.append(['disableHost', [name], {}])

+     return changes

+ 

+ 

+ def get_host_updates(o_idx, n_idx):

+     common = set(n_idx) & set(o_idx)

+     changes = []

+     for name in common:

+         host = n_idx[name]

+         orig = o_idx[name]

+         # fields we care about: arches, capacity, description, comment, enabled

+         #  channels

+         edits = {}

+         for key in 'arches', 'capacity', 'description', 'comment':

+             if host[key] != orig[key]:

+                 edits[key] = host[key]

+         if edits:

+             changes.append(['editHost', [name], edits])

+         ochan = set([c['name'] for c in orig['channels']])

+         nchan = set([c['name'] for c in host['channels']])

+         for chan in nchan - ochan:

+             changes.append(['addHostToChannel', [name, chan], {}])

+         for chan in ochan - nchan:

+             changes.append(['removeHostFromChannel', [name, chan], {}])

+         # new host entry will be enabled by default

+         if host['enabled'] != orig['enabled']:

+             if host['enabled']:

+                 changes.append(['enableHost', [name], {}])

+             else:

+                 changes.append(['disableHost', [name], {}])

+ 

+     return changes

+ 

+ 

+ def print_changes(changes):

+     import pprint

+     pprint.pprint(changes)

+     # TODO: better output

+ 

+ 

+ def do_changes(changes):

+     session.multicall = True

+     for method, args, kw in changes:

+         session.callMethod(method, *args, **kw)

+     session.multiCall(strict=True)

+ 

+ 

+ def get_host_data():

+     hosts = session.listHosts()

+ 

+     # also fetch channels

+     session.multicall = True

+     for host in hosts:

+         session.listChannels(hostID=host['id'])

+     for host, [channels] in zip(hosts, session.multiCall(strict=True)):

+         host['channels'] = channels

+ 

+     return hosts

+ 

+ 

+ if __name__ == '__main__':

+     main()

This pair of scripts can dump current host data to a file and restore it later.

1 new commit added

  • koji-restore-hosts: also restore enabled state
6 years ago

Thanks Mike. Looks good. What are the scenarios where a user would use these tools? Can we put that context into the README?

Thanks Mike. Looks good. What are the scenarios where a user would use these tools? Can we put that context into the README?

My current use case is to save and restore builder state on a stage instance when resetting the stage db, but I can imagine other uses. An admin might for example dump the host config nightly providing an easy way to restore that data in the event of administration error.

An admin might dump before making some temporary mass changes to hosts (e.g lots of disables, or channel adjustments), providing a quick way to reset to the previous state.

Since host data is now versioned, I want to eventually allow dumping this data at an event, but the current api does not offer the option. When that happens, you could easily mass-revert host changes without having to make the dump beforehand.

Cool, that's great context. Thanks. If we merge this, I can add some docs to --help.

It reminds me of the koji_host module, https://github.com/ktdreyer/koji-ansible#koji_host

Commit 34a57e5 fixes this pull-request

Pull-Request has been merged by mikem

6 years ago