From be2fae90ae8c7e697f604d0cae47ab6aa9768558 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Aug 25 2017 20:48:02 +0000 Subject: move custom xmlrpc marshaller to its own file --- diff --git a/devtools/fakehub b/devtools/fakehub index 4e4ba07..ccb5f73 100755 --- a/devtools/fakehub +++ b/devtools/fakehub @@ -14,6 +14,7 @@ sys.path.insert(0, os.getcwd()) sys.path.insert(1, os.path.join(os.getcwd(), 'hub')) import koji import kojixmlrpc +import koji.xmlrpcplus def get_url(environ): @@ -57,7 +58,7 @@ def get_request(): else: args.append(nice_literal(s)) args = koji.encode_args(*args, **kwargs) - request = koji.dumps(args, method, allow_none=1) + request = koji.xmlrpcplus.dumps(args, method, allow_none=1) return request @@ -67,7 +68,7 @@ def start_response(status, headers): def parse_response(data): - p, u = koji.getparser() + p, u = koji.xmlrpcplus.getparser() for chunk in data: p.feed(chunk) p.close() diff --git a/hub/kojihub.py b/hub/kojihub.py index 8c355b4..276f30f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -55,6 +55,8 @@ import time import types import xmlrpclib import zipfile + +import koji.xmlrpcplus from koji.context import context try: @@ -559,7 +561,7 @@ def make_task(method, arglist, **opts): raise koji.GenericError("invalid channel policy") # encode xmlrpc request - opts['request'] = koji.dumps(tuple(arglist), methodname=method, allow_none=1) + opts['request'] = koji.xmlrpcplus.dumps(tuple(arglist), methodname=method, allow_none=1) opts['state'] = koji.TASK_STATES['FREE'] opts['method'] = method koji.plugin.run_callbacks('preTaskStateChange', attribute='state', old=None, new='FREE', info=opts) diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index 29c2463..4692a76 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -35,7 +35,7 @@ import koji.plugin import koji.policy import koji.util # import xmlrpclib functions from koji to use tweaked Marshaller -from koji import getparser, dumps, Fault +from koji.xmlrpcplus import getparser, dumps, Fault from koji.context import context diff --git a/koji/__init__.py b/koji/__init__.py index 9ecdfdf..4d5ea76 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -73,54 +73,12 @@ import struct import tempfile import time import traceback -import types from . import util import warnings import xml.sax import xml.sax.handler import six.moves.urllib -from six.moves.xmlrpc_client import getparser, loads, dumps, Fault - -# Workaround to allow xmlrpclib deal with iterators and 64-bit ints -class Marshaller(six.moves.xmlrpc_client.Marshaller): - - dispatch = six.moves.xmlrpc_client.Marshaller.dispatch.copy() - - def dump_generator(self, value, write): - dump = self.__dump - write("\n") - for v in value: - dump(v, write) - write("\n") - dispatch[types.GeneratorType] = dump_generator - - def dump_datetime(self, value, write): - # For backwards compatibility, we return datetime objects as strings - value = value.isoformat(' ') - self.dump_string(value, write) - dispatch[datetime.datetime] = dump_datetime - - MAXI8 = 2 ** 64 - 1 - MINI8 = -2 ** 64 - def dump_i8(self, value, write): - # python2's xmlrpclib doesn't support i8 extension for marshalling, - # but can unmarshall it correctly. - if value > Marshaller.MAXI8 or value < Marshaller.MINI8: - raise OverflowError, "long int exceeds XML-RPC limits" - elif value > six.moves.xmlrpc_client.MAXINT or \ - value < six.moves.xmlrpc_client.MININT: - write("") - write(str(int(value))) - write("\n") - else: - write("") - write(str(int(value))) - write("\n") - dispatch[types.LongType] = dump_i8 - dispatch[types.IntType] = dump_i8 - -six.moves.xmlrpc_client.Marshaller = Marshaller -six.moves.xmlrpc_client.FastMarshaller = None +from koji.xmlrpcplus import getparser, loads, dumps, Fault PROFILE_MODULES = {} # {module_name: module_instance} diff --git a/koji/daemon.py b/koji/daemon.py index 71f8bf0..1947dd3 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -22,6 +22,7 @@ import koji import koji.tasks +import koji.xmlrpcplus from koji.tasks import safe_rmtree from koji.util import md5_constructor, adler32_constructor, parseStatus, dslice import os @@ -1208,12 +1209,12 @@ class TaskManager(object): try: response = (handler.run(),) # note that we wrap response in a singleton tuple - response = koji.dumps(response, methodresponse=1, allow_none=1) + response = koji.xmlrpcplus.dumps(response, methodresponse=1, allow_none=1) self.logger.info("RESPONSE: %r" % response) self.session.host.closeTask(handler.id, response) return - except koji.Fault, fault: - response = koji.dumps(fault) + except koji.xmlrpcplus.Fault, fault: + response = koji.xmlrpcplus.dumps(fault) tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n") self.logger.warn("FAULT:\n%s" % tb) except (SystemExit, koji.tasks.ServerExit, KeyboardInterrupt): @@ -1232,7 +1233,7 @@ class TaskManager(object): if issubclass(e_class, koji.GenericError): #just pass it through tb = str(e) - response = koji.dumps(koji.Fault(faultCode, tb)) + response = koji.xmlrpcplus.dumps(koji.xmlrpcplus.Fault(faultCode, tb)) # if we get here, then we're handling an exception, so fail the task self.session.host.failTask(handler.id, response) diff --git a/koji/xmlrpcplus.py b/koji/xmlrpcplus.py new file mode 100644 index 0000000..9505200 --- /dev/null +++ b/koji/xmlrpcplus.py @@ -0,0 +1,109 @@ +""" +Custom xmlrpc handling for Koji +""" + +import six +import six.moves.xmlrpc_client as xmlrpc_client +import types + + +# duplicate a few values that we need +getparser = xmlrpc_client.getparser +loads = xmlrpc_client.loads +Fault = xmlrpc_client.Fault + + +class ExtendedMarshaller(xmlrpc_client.Marshaller): + + dispatch = xmlrpc_client.Marshaller.dispatch.copy() + + def dump_generator(self, value, write): + dump = self.__dump + write("\n") + for v in value: + dump(v, write) + write("\n") + dispatch[types.GeneratorType] = dump_generator + + MAXI8 = 2 ** 64 - 1 + MINI8 = -2 ** 64 + + def dump_i8(self, value, write): + # python2's xmlrpclib doesn't support i8 extension for marshalling, + # but can unmarshall it correctly. + if (value > self.MAXI8 or value < self.MINI8): + raise OverflowError("long int exceeds XML-RPC limits") + elif (value > xmlrpc_client.MAXINT or + value < xmlrpc_client.MININT): + write("") + write(str(int(value))) + write("\n") + else: + write("") + write(str(int(value))) + write("\n") + dispatch[types.LongType] = dump_i8 + dispatch[types.IntType] = dump_i8 + + # we always want to allow None + def dump_nil(self, value, write): + write("") + dispatch[type(None)] = dump_nil + + +def dumps(params, methodname=None, methodresponse=None, encoding=None, + allow_none=1, marshaller=None): + """encode an xmlrpc request or response + + Differences from the xmlrpclib version: + - allow_none is on by default + - uses our ExtendedMarshaller by default + - option to specify marshaller + """ + + if isinstance(params, Fault): + methodresponse = 1 + elif not isinstance(params, tuple): + raise TypeError('params must be a tuple of Fault instance') + elif methodresponse and len(params) != 1: + raise ValueError('response tuple must be a singleton') + + if not encoding: + encoding = "utf-8" + + if marshaller is not None: + m = marshaller(encoding) + else: + m = ExtendedMarshaller(encoding, allow_none) + + data = m.dumps(params) + + if encoding != "utf-8": + xmlheader = "\n" % str(encoding) + else: + xmlheader = "\n" # utf-8 is default + + # standard XML-RPC wrappings + if methodname: + # a method call + if six.PY2 and isinstance(methodname, six.text_type): + # Do we need this? + methodname = methodname.encode(encoding, 'xmlcharrefreplace') + parts = ( + xmlheader, + "\n" + "", methodname, "\n", + data, + "\n" + ) + elif methodresponse: + # a method response, or a fault structure + parts = ( + xmlheader, + "\n", + data, + "\n" + ) + else: + return data # return as is + return ''.join(parts)