From be0e85085967b9f601c1219b3277f12d7745bafc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20Hor=C3=A1k?= Date: Tue, 26 May 2020 19:00:51 +0200 Subject: [koji PATCH] add more debugging output --- koji/__init__.py | 18 +++++++++++++----- koji/tasks.py | 18 +++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/koji/__init__.py b/koji/__init__.py index 9f516add..745013bd 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1678,18 +1678,26 @@ def openRemoteFile(relpath, topurl=None, topdir=None, tempdir=None): fo = tempfile.TemporaryFile(dir=tempdir) resp = requests.get(url, stream=True) length = 0 + cnt = 0 + csize = 8192 logger = logging.getLogger("koji.build") try: - for chunk in resp.iter_content(chunk_size=8192): - length += len(chunk) - fo.write(chunk) + for chunk in resp.iter_content(chunk_size=csize): + cnt += 1 + clen = len(chunk) + length += clen + wlen = fo.write(chunk) + if clen != wlen: + logger.info("openRemoteFile: %s, chunk number: %d, clen != wlen (%d != %d)" % (url, cnt, clen, wlen)) + if clen != csize: + logger.info("openRemoteFile: %s, chunk number: %d, clen != csize (%d != %d)" % (url, cnt, clen, csize)) finally: header_length = int(resp.headers.get('content-length'), 0) if length != header_length: - logger.info("openRemoteFile: %s, header: %d, got: %d" % (url, header_length, length)) + logger.info("openRemoteFile: %s, header: %d, got: %d, chunk count: %d, expected count: %d" % (url, header_length, length, cnt, (header_length + csize - 1)/csize)) logger.info(resp.headers) else: - logger.info("openRemoteFile: %s [success], size %d" % (url, length)) + logger.info("openRemoteFile: %s [success], size %d, chunk count: %d" % (url, length, cnt)) resp.close() fo.seek(0) elif topdir: diff --git a/koji/tasks.py b/koji/tasks.py index 392aab61..c6c160ca 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -480,20 +480,28 @@ class BaseTaskHandler(object): url = "%s/%s" % (self.options.topurl, relpath) resp = requests.get(url, stream=True) length = 0 + cnt = 0 + csize = 8192 try: if not os.path.exists(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn)) with open(fn, 'wb') as fdst: - for chunk in resp.iter_content(chunk_size=8192): - length += len(chunk) - fdst.write(chunk) + for chunk in resp.iter_content(chunk_size=csize): + cnt += 1 + clen = len(chunk) + length += clen + wlen = fdst.write(chunk) + if clen != wlen: + self.logger.info("localPath: %s, chunk number: %d, clen != wlen (%d != %d)" % (url, cnt, clen, wlen)) + if clen != csize: + self.logger.info("localPath: %s, chunk number: %d, clen != csize (%d != %d)" % (url, cnt, clen, csize)) finally: header_length = int(resp.headers.get('content-length'), 0) if length != header_length: - self.logger.info("localPath: %s, header: %d, got: %d" % (url, header_length, length)) + self.logger.info("localPath: %s, header: %d, got: %d, chunk count: %d, expected count: %d" % (url, header_length, length, cnt, (header_length + csize - 1)/csize)) self.logger.info(resp.headers) else: - self.logger.info("localPath: %s [success], size %d" % (url, length)) + self.logger.info("localPath: %s [success], size %d, chunk count: %d" % (url, length, cnt)) resp.close() else: fn = "%s/%s" % (self.options.topdir, relpath) -- 2.21.3