+ from coprs.request import get_request_class

+ app.request_class = get_request_class(app)

+ 

+ from coprs.request import save_form_file_field_to

+     def create_new_from_upload(cls, user, copr, form_field, orig_filename,

+         :param form_field: object representing an uploaded file in form

+             save_form_file_field_to(form_field, file_path)

+ """

+ Use NamedTemporaryFile for large file uploads, so we eventually don't have copy

+ the uploaded file to the final destination.

+ """

+ 

+ import os

+ import tempfile

+ import typing as t

+ 

+ from werkzeug.formparser import default_stream_factory

+ import flask

+ 

+ 

+ NAMED_FILE_FROM_BYTES = 50*1024*1024

+ 

+ 

+ def get_request_class(app):

+     """

+     Return a Copr-specific Request class instantiated by Flask for every

+     request.  We want to override the internal _get_file_stream() (HACK!) method

+     to enforce using the NamedTemporaryFile class for the large uploaded files -

+     instead of the werkzeug's default SpooledTemporaryFile.  While

+     SpooledTemporaryFile() by default stores small files into memory and

+     fallbacks to the /tmp directory, NamedTemporaryFile() data destination can

+     be controlled by our Copr logic, so we can store the file into

+     config[STORAGE_DIR] directly.  The slow uploads (typically network limits,

+     from remote users' boxes) are thus directly stored to the target volume, and

+     once uploaded we don't have to do the expensive cross-volume file copy

+     (/tmp => STORAGE_DIR).  For very large source RPMs this could even mean

+     several minutes.

+ 

+     Be careful, we can't simply "move" the temporary file to the target

+     destination because the upper level Flask/Werkzeug logic automatically

+     unlinks the temporary file when the corresponding request is processed.

+     We have to do os.link() instead.

+ 

+     Reported to Flask upstream: https://github.com/pallets/flask/issues/4844

+     """

+ 

+     class _CoprRequestClass(flask.Request):

+         # pylint: disable=no-self-use

+         def _get_file_stream(

+             self,

+             total_content_length: t.Optional[int],

+             content_type: t.Optional[str],

+             filename: t.Optional[str] = None,

+             content_length: t.Optional[int] = None,

+         ) -> t.IO[bytes]:

+             """Called to get a stream for the file upload."""

+ 

+             # We do this hack only for reasonably large uploaded files,

+             # elsewhere we use the default Flask upload mechanism.  This is to

+             # stay as close to the upstream behavior as possible and stay

+             # compatible in the future.

+             if total_content_length and total_content_length >= NAMED_FILE_FROM_BYTES:

+                 # pylint: disable=consider-using-with

+                 tfile = tempfile.NamedTemporaryFile(

+                     "rb+", dir=app.config["STORAGE_DIR"])

+                 setattr(tfile, "named_file", True)

+                 return t.cast(t.IO[bytes], tfile)

+ 

+             # For the smaller RPMs, we keep using the default Temporary files

+             # abstraction.

+             return default_stream_factory(

+                 total_content_length,

+                 content_type,

+                 filename,

+                 content_length,

+             )

+ 

+     return _CoprRequestClass

+ 

+ 

+ def save_form_file_field_to(field, path: str):

+     """

+     Store the uploaded data on Werkzeug file field.  Use this if the hack

+     with _CoprRequestClass above is in action, and when you are sure that the

+     uploaded file is on the same partition with the target path (so we can

+     hardlink).

+     """

+     stream = field.data.stream

+ 

+     # Detect if SpooledTemporaryFile or NamedTemporaryFile is the storage

+     if hasattr(stream, "named_file"):

+         os.link(stream.name, path)

+     else:

+         field.data.save(path)

+             form.pkgs,

+             form.pkgs,

+         for key in content:

+             if isinstance(content[key], FileStorage):

+                 method = self.test_class_object.post_api3_with_auth_multipart

+                 break

+     def submit_uploaded_build(self, project, source_rpm, build_options=None):

+         """

+         Submit a build via source RPM

+         """

+         route = "/api_3/build/create/upload"

+         data = {

+             "ownername": self.transaction_username,

+             "projectname": project,

+             "pkgs": FileStorage(

+                 stream=open(source_rpm, "rb"),

+                 filename=os.path.basename(source_rpm),

+                 content_type="application/rpm",

+             )

+         }

+         data.update(self._form_data_from_build_options(build_options))

+         return self.post(route, data)

+ 

+ import tempfile

+ from unittest import mock

+ from coprs.request import NAMED_FILE_FROM_BYTES

+     @mock.patch('coprs.logic.builds_logic.save_form_file_field_to')

+     def test_create_new_from_upload_no_space_left(self, save_form_patch):

+         save_form_patch.side_effect = OSError("[Errno 28] No space left on device")

+             BuildsLogic.create_new_from_upload(self.u1, self.c1, None,

+     # Call SpooledTemporaryFile twice (once in memory, once in file), and

+     # NamedTemporaryFile once.

+     @pytest.mark.parametrize("kbytes", [100, 1024, 100*1024])

+     def test_srpm_upload(self, kbytes):

+         self.web_ui.new_project("test", ["fedora-rawhide-i386"])

+ 

+         srpm_base = "test.src.rpm"

+         workdir = os.path.dirname(__file__)

+         srpm = os.path.join(workdir, srpm_base)

+         with open(srpm, "w", encoding="utf-8") as fd:

+             string = "x"*1024*kbytes

+             fd.write(string)

+ 

+         not_patched = tempfile.NamedTemporaryFile

+         with mock.patch("coprs.request.tempfile.NamedTemporaryFile",

+                         wraps=not_patched) as patch:

+             resp = self.api3.submit_uploaded_build("test", srpm)

+             exp_count = 1 if kbytes*1024 > NAMED_FILE_FROM_BYTES else 0

+             assert patch.call_count == exp_count

+ 

+         assert resp.status_code == 200

+         # url = https://copr.stg.fedoraproject.org/tmp/tmpf_3w8r9i/test.src.rpm'

+         url = json.loads(resp.data)["source_package"]["url"]

+         tmpdir = url.split("/")[-2]

+         storage = os.path.join(self.app.config["STORAGE_DIR"], tmpdir, srpm_base)

+         stat = os.stat(storage)

+         assert stat.st_size == 1024*kbytes

+         assert stat.st_nlink == 1

+ 

+     @TransactionDecorator("u1")

+     @pytest.mark.usefixtures("f_users", "f_users_api", "f_mock_chroots", "f_db")

+         def save_field(_field, file_path):

+             with open(file_path, "w", encoding="utf-8") as fd:

+ 

+         with mock.patch("coprs.logic.builds_logic.save_form_file_field_to",

+                         new=save_field):

+             build = BuildsLogic.create_new_from_upload(

+                 user, copr, None, os.path.basename(filename),

+                 chroot_names=["fedora-18-x86_64"],

+                 copr_dirname=None)