koji

Clone

#3785 track update time in host table
Closed 2 years ago by tkopecek. Opened 2 years ago by mikem.
mikem/koji host-update-ts  into  master

file modified
+37 -20 
@@ -1,6 +1,7 @@ 
 

  from __future__ import absolute_import, division
 

  

  import ast
 

+ import dateutil.parser
 

  import fnmatch
 

  import itertools
 

  import json
 
@@ -3136,24 +3137,14 @@ 
 

          else:
 

              return 'N'
 

  

-     try:
 

-         first = session.getLastHostUpdate(hosts[0]['id'], ts=True)
 

-         opts = {'ts': True}
 

-     except koji.ParameterError:
 

-         # Hubs prior to v1.25.0 do not have a "ts" parameter for getLastHostUpdate
 

-         first = session.getLastHostUpdate(hosts[0]['id'])
 

-         opts = {}
 

- 

-     # pull in the last update using multicall to speed it up a bit
 

-     with session.multicall() as m:
 

-         result = [m.getLastHostUpdate(host['id'], **opts) for host in hosts[1:]]
 

-     updateList = [first] + [x.result for x in result]
 

+     if 'update_ts' not in hosts[0]:
 

+         _get_host_update_oldhub(session, hosts)
 

  

-     for host, update in zip(hosts, updateList):
 

-         if update is None:
 

+     for host in hosts:
 

+         if host['update_ts'] is None:
 

              host['update'] = '-'
 

          else:
 

-             host['update'] = koji.formatTimeLong(update)
 

+             host['update'] = koji.formatTimeLong(host['update_ts'])
 

          host['enabled'] = yesno(host['enabled'])
 

          host['ready'] = yesno(host['ready'])
 

          host['arches'] = ','.join(host['arches'].split())
 
@@ -3203,6 +3194,33 @@ 
 

          print(mask % host)
 

  

  

+ def _get_host_update_oldhub(session, hosts):
 

+     """Fetch host update times from older hubs"""
 

+ 

+     # figure out if hub supports ts parameter
 

+     try:
 

+         first = session.getLastHostUpdate(hosts[0]['id'], ts=True)
 

+         opts = {'ts': True}
 

+     except koji.ParameterError:
 

+         # Hubs prior to v1.25.0 do not have a "ts" parameter for getLastHostUpdate
 

+         first = session.getLastHostUpdate(hosts[0]['id'])
 

+         opts = {}
 

+ 

+     with session.multicall() as m:
 

+         result = [m.getLastHostUpdate(host['id'], **opts) for host in hosts[1:]]
 

+ 

+     updateList = [first] + [x.result for x in result]
 

+ 

+     for host, update in zip(hosts, updateList):
 

+         if 'ts' in opts:
 

+             host['update_ts'] = update
 

+         elif update is None:
 

+             host['update_ts'] = None
 

+         else:
 

+             dt = dateutil.parser.parse(update)
 

+             host['update_ts'] = time.mktime(dt.timetuple())
 

+ 

+ 

  def anon_handle_list_pkgs(goptions, session, args):
 

      "[info] Print the package listing for tag or for owner"
 

      usage = "usage: %prog list-pkgs [options]"
 
@@ -3643,11 +3661,10 @@ 
 

              print("Comment:")
 

          print("Enabled: %s" % (info['enabled'] and 'yes' or 'no'))
 

          print("Ready: %s" % (info['ready'] and 'yes' or 'no'))
 

-         try:
 

-             update = session.getLastHostUpdate(info['id'], ts=True)
 

-         except koji.ParameterError:
 

-             # Hubs prior to v1.25.0 do not have a "ts" parameter for getLastHostUpdate
 

-             update = session.getLastHostUpdate(info['id'])
 

+ 

+         if 'update_ts' not in info:
 

+             _get_host_update_oldhub(session, [info])
 

+         update = info['update_ts']
 

          if update is None:
 

              update = "never"
 

          else:

file modified
+1 
@@ -159,6 +159,7 @@ 
 

  	id SERIAL NOT NULL PRIMARY KEY,
 

  	user_id INTEGER NOT NULL REFERENCES users (id),
 

  	name VARCHAR(128) UNIQUE NOT NULL,
 

+ 	update_time TIMESTAMPTZ,

worth an index?

 
  	task_load FLOAT CHECK (NOT task_load < 0) NOT NULL DEFAULT 0.0,
 

  	ready BOOLEAN NOT NULL DEFAULT 'false'
 

  ) WITHOUT OIDS;

file modified
+19 -8 
@@ -2551,7 +2551,7 @@ 
 

              'expired IS FALSE',
 

              'master IS NULL',
 

              'active IS TRUE',
 

-             "update_time > NOW() - '5 minutes'::interval"
 

+             "sessions.update_time > NOW() - '5 minutes'::interval"
 

          ],
 

          joins=[
 

              'sessions USING (user_id)',
 
@@ -5483,6 +5483,7 @@ 
 

      - id
 

      - user_id
 

      - name
 

+     - update_ts
 

      - arches
 

      - task_load
 

      - capacity
 
@@ -5498,6 +5499,7 @@ 
 

          ('host.id', 'id'),
 

          ('host.user_id', 'user_id'),
 

          ('host.name', 'name'),
 

+         ("date_part('epoch', host.update_time)", 'update_ts'),
 

          ('host.ready', 'ready'),
 

          ('host.task_load', 'task_load'),
 

          ('host_config.arches', 'arches'),
 
@@ -13283,6 +13285,7 @@ 
 

              ('host.id', 'id'),
 

              ('host.user_id', 'user_id'),
 

              ('host.name', 'name'),
 

+             ("date_part('epoch', host.update_time)", 'update_ts'),
 

              ('host.ready', 'ready'),
 

              ('host.task_load', 'task_load'),
 

              ('host_config.arches', 'arches'),
 
@@ -13302,9 +13305,14 @@ 
 

  

          The timestamp represents the last time the host with the given
 

          ID contacted the hub. Returns None if the host has never contacted
 

-         the hub."""
 

+         the hub.
 

+ 

+         The timestamp returned here may be different than the newer
 

+         update_ts field now returned by the getHost and listHosts calls.
 

+         """
 

          opts = {'order': '-update_time', 'limit': 1}
 

-         query = QueryProcessor(tables=['sessions'], columns=['update_time'],
 

+         query = QueryProcessor(tables=['sessions'], columns=['sessions.update_time'],
 

+                                aliases=['update_time'],
 

                                 joins=['host ON sessions.user_id = host.user_id'],
 

                                 clauses=['host.id = %(hostID)i'], values={'hostID': hostID},
 

                                 opts=opts)
 
@@ -14256,11 +14264,14 @@ 
 

      def updateHost(self, task_load, ready):
 

          host_data = get_host(self.id)
 

          task_load = float(task_load)
 

-         if task_load != host_data['task_load'] or ready != host_data['ready']:
 

-             update = UpdateProcessor('host', clauses=['id=%(id)i'], values={'id': self.id},
 

-                                      data={'task_load': task_load, 'ready': ready})
 

-             update.execute()
 

-             context.commit_pending = True
 

+         update = UpdateProcessor(
 

+                 'host',
 

+                  data={'task_load': task_load, 'ready': ready},
 

+                  rawdata={'update_time': 'NOW()'},
 

+                  clauses=['id=%(id)i'],
 

+                  values={'id': self.id},
 

+         )
 

+         update.execute()
 

  

      def getLoadData(self):
 

          """Get load balancing data

file modified
+4 -1 
@@ -34,6 +34,7 @@ 
 

                           'comment': 'test-comment',
 

                           'description': 'test-description'}
 

          self.last_update = 1615875554.862938
 

+         self.last_update_str = '2021-03-16 06:19:14.862938-00:00'
 

          self.list_channels = [{'id': 1, 'name': 'default'}, {'id': 2, 'name': 'createrepo'}]
 

          self.error_format = """Usage: %s hostinfo [options] <hostname> [<hostname> ...]
 

  (Specify the --help global option for a list of other help options)
 
@@ -142,6 +143,7 @@ 
 

  

      @mock.patch('sys.stdout', new_callable=StringIO)
 

      def test_hostinfo_valid_param_error(self, stdout):
 

+         '''Test the fallback code for getting timestamps from old hubs'''
 

          expected = """Name: kojibuilder
 

  ID: 1
 

  Arches: x86_64
 
@@ -157,7 +159,8 @@ 
 

  None
 

  """
 

          self.session.getHost.return_value = self.hostinfo
 

-         self.session.getLastHostUpdate.side_effect = [koji.ParameterError, self.last_update]
 

+         # simulate an older hub that doesn't support the ts option for getLastHostUpdate
 

+         self.session.getLastHostUpdate.side_effect = [koji.ParameterError, self.last_update_str]
 

          self.session.listChannels.return_value = self.list_channels
 

          rv = anon_handle_hostinfo(self.options, self.session, [self.hostinfo['name']])
 

          self.assertEqual(rv, None)

file modified
+3 -1 
@@ -179,10 +179,12 @@ 
 

  

      @mock.patch('sys.stdout', new_callable=StringIO)
 

      def test_list_hosts_param_error_get_last_host_update(self, stdout):
 

-         host_update = 1615875554.862938
 

+         # host_update = 1615875554.862938
 

+         host_update = '2021-03-16 06:19:14.862938-00:00'
 

          expected = "kojibuilder N   Y    0.0/2.0  x86_64           " \
 

                     "Tue, 16 Mar 2021 06:19:14 UTC      \n"
 

  

+         # simulate an older hub that doesn't support the ts option for getLastHostUpdate
 

          self.session.getLastHostUpdate.side_effect = [koji.ParameterError, host_update]
 

          self.session.multiCall.return_value = [[host_update]]
 

          self.session.listHosts.return_value = self.list_hosts

file modified
+6 -4 
@@ -33,12 +33,13 @@ 
 

  

          self.assertEqual(len(self.queries), 1)
 

          query = self.queries[0]
 

-         columns = ['host.id', 'host.user_id', 'host.name', 'host.ready',
 

+         columns = ['host.id', 'host.user_id', 'host.name',
 

+                    "date_part('epoch', host.update_time)", 'host.ready',
 

                     'host.task_load', 'host_config.arches',
 

                     'host_config.capacity', 'host_config.description',
 

                     'host_config.comment', 'host_config.enabled']
 

          joins = ['host ON host.id = host_config.host_id']
 

-         aliases = ['id', 'user_id', 'name', 'ready', 'task_load',
 

+         aliases = ['id', 'user_id', 'name', 'update_ts', 'ready', 'task_load',
 

                     'arches', 'capacity', 'description', 'comment', 'enabled']
 

          clauses = ['(host_config.active = TRUE)', '(host.name = %(host_name)s)']
 

          values = {'host_name': 'hostname'}
 
@@ -54,12 +55,13 @@ 
 

  

          self.assertEqual(len(self.queries), 1)
 

          query = self.queries[0]
 

-         columns = ['host.id', 'host.user_id', 'host.name', 'host.ready',
 

+         columns = ['host.id', 'host.user_id', 'host.name',
 

+                    "date_part('epoch', host.update_time)", 'host.ready',
 

                     'host.task_load', 'host_config.arches',
 

                     'host_config.capacity', 'host_config.description',
 

                     'host_config.comment', 'host_config.enabled']
 

          joins = ['host ON host.id = host_config.host_id']
 

-         aliases = ['id', 'user_id', 'name', 'ready', 'task_load',
 

+         aliases = ['id', 'user_id', 'name', 'update_ts', 'ready', 'task_load',
 

                     'arches', 'capacity', 'description', 'comment', 'enabled']
 

          clauses = ['(host_config.create_event <= 345 AND ( host_config.revoke_event IS NULL '
 

                     'OR 345 < host_config.revoke_event ))',

file modified
+11 -5 
@@ -1713,11 +1713,17 @@ 
 

  

      values['channels'] = server.listChannels()
 

  

-     with server.multicall() as m:
 

-         updates = [m.getLastHostUpdate(host['id'], ts=True) for host in hosts]
 

- 

-     for host, lastUpdate in zip(hosts, updates):
 

-         host['last_update'] = lastUpdate.result
 

+     if hosts and 'update_ts' not in hosts[0]:
 

+         # be nice with older hub
 

+         # TODO remove this compat workaround after a release
 

+         with server.multicall() as m:
 

+             updates = [m.getLastHostUpdate(host['id'], ts=True) for host in hosts]
 

+ 

+         for host, lastUpdate in zip(hosts, updates):
 

+             host['last_update'] = lastUpdate.result
 

+     else:
 

+         for host in hosts:
 

+             host['last_update'] = koji.formatTimeLong(host['update_ts'])
 

  

      # Paginate after retrieving last update info so we can sort on it
 

      kojiweb.util.paginateList(values, hosts, start, 'hosts', 'host', order)

Store an update_time in the host table each time the host checks in, and use this instead of the session update_time.

WIP
I still need to fix some unit tests
Edited 2 years ago by mikem

1 new commit added

  • fix unit tests
2 years ago

worth an index?

migration script
Edited 2 years ago by tkopecek

conflicts with session.update_time:

psycopg2.errors.AmbiguousColumn: column reference "update_time" is ambiguous
LINE 11:    AND (update_time > NOW() - '5 minutes'::interval)

1 new commit added

  • avoid ambiguous columns refs
2 years ago

Because in this PR is change in schema.sql, we should add it to upgrade-schema sql also, or not? Tomas mentioned it also in comments.

Added in #3819

Pull-Request has been closed by tkopecek
2 years ago
Metadata
None
No Tags
Changes Summary 7
+37 -20
file changed
cli/koji_cli/commands.py
+1 -0
file changed
docs/schema.sql
+19 -8
file changed
kojihub/kojihub.py
+4 -1
file changed
tests/test_cli/test_hostinfo.py
+3 -1
file changed
tests/test_cli/test_list_hosts.py
+6 -4
file changed
tests/test_hub/test_get_host.py
+11 -5
file changed
www/kojiweb/index.py
Powered by Pagure 5.14.1
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.