From da038077047ba93c5d38fad8163fe37899c32f53 Mon Sep 17 00:00:00 2001 From: Luke Macken Date: Feb 03 2016 17:12:39 +0000 Subject: Initial commit --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0290220 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.pyc +*.egg-info +*.egg* +development.ini +dist +*.crt +*.key +*.gpg +fedmsg.d/outhouse.py +outhouse/tests/vcr-request-data +.coverage +*.swp diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4362b49 --- /dev/null +++ b/LICENSE @@ -0,0 +1,502 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..563287e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +graft fedmsg.d +recursive-include outhouse/tests/handler_tests/data * +recursive-include outhouse/tests/vcr-request-data * +include *.rst +include LICENSE +include test-requirements.txt diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..407ef1d --- /dev/null +++ b/README.rst @@ -0,0 +1,68 @@ +Outhouse +======== + +This is a backend daemon that listens to fedmsg and queries PDC +to determine which artifacts are affected. It will then trigger the +appropriate rebuilds of these components. + +Development +----------- + +Set up a python virtualenv:: + + $ sudo dnf -y install python-virtualenvwrapper + $ source ~/.bashrc + $ mkvirtualenv outhouse + +Setup outhouse and its dependencies:: + + $ workon outhouse + $ pip install -r requirements.txt + $ python setup.py develop + +Try running the test suite:: + + $ pip install -r test-requirements.txt + $ nosetests + +Check test suite coverage:: + + $ pip install coverage + $ nosetests --with-coverage --cover-package=pdcupdater + + +Getting an authentication token +------------------------------- + +...from https://pdc.fedoraproject.org/ + +- go to https://pdc.fedoraproject.org/ in your browser and login. +- go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/ +- open up the devtools console in your browser, and find the request for the current page. +- right click to open a context menu and select 'copy as cURL' +- paste that into a terminal. It should have your saml cookie. +- before hitting enter, edit the command to add the following option: + + - ``-H 'Accept: application/json'``, to tell the API you want data + +- the command should print out your token. + +Copy ``fedmsg.d/pdcupdater-example.py`` to ``fedmsg.d/pdcupdater.py`` and fill +in your token there. + +Running the fedmsg-hub +---------------------- + +Run it for real:: + + $ fedmsg-hub + +Finally, you can take the dev instructions from `the-new-hotness +`_ and learn how to +set up a local fedmsg-relay so you can replay production messages and more +fully test out outhouse. + + +A fake bodhi update can then be sent to test the consumer:: + + echo '{"update": {"builds": [{"nvr": "kernel-4.3.4-200.fc24"}], "release": {"long_name": "Fedora 24"}}}' | fedmsg-logger --modname bodhi --topic update.complete.stable --json-input diff --git a/fedmsg.d/base.py b/fedmsg.d/base.py new file mode 100644 index 0000000..425ff08 --- /dev/null +++ b/fedmsg.d/base.py @@ -0,0 +1,54 @@ +# This file is part of fedmsg. +# Copyright (C) 2012 Red Hat, Inc. +# +# fedmsg is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# fedmsg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with fedmsg; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: Ralph Bean +# +config = dict( + # Set this to dev if you're hacking on fedmsg or an app. + # Set to stg or prod if running in the Fedora Infrastructure + environment="prod", + + # Default is 0 + high_water_mark=0, + io_threads=1, + + ## For the fedmsg-hub and fedmsg-relay. ## + + # We almost always want the fedmsg-hub to be sending messages with zmq as + # opposed to amqp or stomp. + zmq_enabled=True, + + # When subscribing to messages, we want to allow splats ('*') so we tell + # the hub to not be strict when comparing messages topics to subscription + # topics. + zmq_strict=False, + + # Number of seconds to sleep after initializing waiting for sockets to sync + post_init_sleep=0.5, + + # Wait a whole second to kill all the last io threads for messages to + # exit our outgoing queue (if we have any). This is in milliseconds. + zmq_linger=1000, + + # See the following + # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html + # - http://api.zeromq.org/3-2:zmq-setsockopt + zmq_tcp_keepalive=1, + zmq_tcp_keepalive_cnt=3, + zmq_tcp_keepalive_idle=60, + zmq_tcp_keepalive_intvl=5, +) diff --git a/fedmsg.d/logging.py b/fedmsg.d/logging.py new file mode 100644 index 0000000..89f27d4 --- /dev/null +++ b/fedmsg.d/logging.py @@ -0,0 +1,40 @@ +# Setup fedmsg logging. +# See the following for constraints on this format http://bit.ly/Xn1WDn +bare_format = "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" + +config = dict( + logging=dict( + version=1, + formatters=dict( + bare={ + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": bare_format + }, + ), + handlers=dict( + console={ + "class": "logging.StreamHandler", + "formatter": "bare", + "level": "DEBUG", + "stream": "ext://sys.stdout", + } + ), + loggers=dict( + fedmsg={ + "level": "DEBUG", + "propagate": False, + "handlers": ["console"], + }, + moksha={ + "level": "DEBUG", + "propagate": False, + "handlers": ["console"], + }, + pkgdb2client={ + "level": "INFO", + "propagate": False, + "handlers": ["console"], + }, + ), + ), +) diff --git a/fedmsg.d/outhouse-example.py b/fedmsg.d/outhouse-example.py new file mode 100644 index 0000000..2bc431f --- /dev/null +++ b/fedmsg.d/outhouse-example.py @@ -0,0 +1,57 @@ + +config = { + # Should we turn on the realtime updater? + 'outhouse.enabled': True, + + # Credentials to talk to PDC + 'outhouse.pdc': { + 'server': 'https://pdc.fedoraproject.org/rest_api/v1/', + 'insecure': False, # Just because we have a self-signed cert in the cloud + 'token': 'AWESOME_SECRET_STRING_GOES_HERE', + # XXX - getting the token is a bit of a pain, but here's a walk through + # 1) go to https://pdc.fedoraproject.org/ in your browser and login. + # 2) go to https://pdc.fedoraproject.org/rest_api/v1/auth/token/obtain/ + # 3) open up the devtools console in your browser, and find the request for the current page. + # 4) right click to open a context menu and select 'copy as cURL' + # 5) paste that into a terminal. It should have your saml cookie. + # 6) before hitting enter, edit the command to add the following option + # -H 'Accept: application/json' # to tell the API you want data + # 7) the command should print out your token. + }, + + ## Credentials to talk to FAS + #'outhouse.fas': { + # 'base_url': 'https://admin.fedoraproject.org/accounts', + # 'username': 'YOUR_USERNAME_GOES_HERE', + # 'password': 'AWESOME_SECRET_PASSWORD_GOES_HERE', + #}, + + # PkgDB details + 'outhouse.pkgdb_url': 'https://admin.fedoraproject.org/pkgdb', + + # Koji details + 'outhouse.koji_url': 'http://koji.fedoraproject.org/kojihub', + + # Where to find composes + 'outhouse.old_composes_url': 'https://kojipkgs.fedoraproject.org/compose/', + + # Where to find the fedora-atomic json definitions. + 'outhouse.fedora_atomic_git_url': 'https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + + # We have an explicit list of these in the config so we can turn them on + # and off individually in production if one is causing an issue. + 'outhouse.handlers': [ + 'outhouse.handlers.atomic:AtomicHandler', + ], + + # Augment the base fedmsg logging config to also handle outhouse loggers. + 'logging': dict( + loggers=dict( + outhouse={ + "level": "DEBUG", + "propagate": False, + "handlers": ["console"], + }, + ) + ) +} diff --git a/fedmsg.d/relay.py b/fedmsg.d/relay.py new file mode 100644 index 0000000..60d4d87 --- /dev/null +++ b/fedmsg.d/relay.py @@ -0,0 +1,15 @@ +config = dict( + endpoints={ + # This is the output side of the relay to which the-new-hotness + # can listen (where the-new-hotness is running as a part of 'fedmsg-hub') + "relay_outbound": [ + "tcp://127.0.0.1:4001", + ], + }, + + # This is the input side of the relay to which 'fedmsg-logger' and 'fedmsg-dg-replay' will send messages. + # It will just repeat those messages out the 'relay_outbound' endpoint on your own box. + relay_inbound=[ + "tcp://127.0.0.1:2003", + ], +) diff --git a/fedmsg.d/ssl.py b/fedmsg.d/ssl.py new file mode 100644 index 0000000..2e701e5 --- /dev/null +++ b/fedmsg.d/ssl.py @@ -0,0 +1,66 @@ +# This file is part of fedmsg. +# Copyright (C) 2012 Red Hat, Inc. +# +# fedmsg is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# fedmsg is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with fedmsg; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Authors: Ralph Bean +# +import os +import socket + +SEP = os.path.sep +here = os.getcwd() + +config = dict( + sign_messages=False, + validate_signatures=False, + + # Use these implementations to sign and validate messages + crypto_backend='x509', + crypto_validate_backends=['x509'], + + ssldir="/etc/pki/fedmsg", + crl_location="https://fedoraproject.org/fedmsg/crl.pem", + crl_cache="/var/run/fedmsg/crl.pem", + crl_cache_expiry=10, + + ca_cert_location="https://fedoraproject.org/fedmsg/ca.crt", + ca_cert_cache="/var/run/fedmsg/ca.crt", + ca_cert_cache_expiry=0, # Never expires + + certnames={ + # In prod/stg, map hostname to the name of the cert in ssldir. + # Unfortunately, we can't use socket.getfqdn() + #"app01.stg": "app01.stg.phx2.fedoraproject.org", + }, + + # A mapping of fully qualified topics to a list of cert names for which + # a valid signature is to be considered authorized. Messages on topics not + # listed here are considered automatically authorized. + routing_policy={ + # Only allow announcements from production if they're signed by a + # certain certificate. + "org.fedoraproject.prod.announce.announcement": [ + "announce-lockbox.phx2.fedoraproject.org", + ], + }, + + # Set this to True if you want messages to be dropped that aren't + # explicitly whitelisted in the routing_policy. + # When this is False, only messages that have a topic in the routing_policy + # but whose cert names aren't in the associated list are dropped; messages + # whose topics do not appear in the routing_policy are not dropped. + routing_nitpicky=False, +) diff --git a/outhouse.spec b/outhouse.spec new file mode 100644 index 0000000..a48ac2a --- /dev/null +++ b/outhouse.spec @@ -0,0 +1,76 @@ +%{!?_licensedir: %global license %%doc} + +%if 0%{?rhel} && 0%{?rhel} <= 6 +%{!?__python2: %global __python2 /usr/bin/python2} +%{!?python2_sitelib: %global python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python2_sitearch: %global python2_sitearch %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} +%endif + +Name: outhouse +Version: 0.0.1 +Release: 1%{?dist} +Summary: Update the product definition center in response to fedmsg + +Group: Development/Libraries +License: LGPLv2+ +URL: https://pagure.io/outhouse +Source0: https://pypi.python.org/packages/source/p/%{name}/%{name}-%{version}.tar.gz +BuildArch: noarch + +BuildRequires: python2-devel +BuildRequires: python-setuptools + +BuildRequires: fedmsg +BuildRequires: python-fedmsg-commands +BuildRequires: python-fedmsg-consumers +BuildRequires: python-requests +BuildRequires: python-dogpile-cache +BuildRequires: python-fedora +BuildRequires: packagedb-cli +BuildRequires: pdc-client + +# For the tests +BuildRequires: python-nose +BuildRequires: python-vcrpy +BuildRequires: python-mock + +Requires: fedmsg +Requires: python-fedmsg-commands +Requires: python-fedmsg-consumers +Requires: python-requests +Requires: python-dogpile-cache +Requires: python-fedora +Requires: packagedb-cli +Requires: pdc-client + +%description +Fedmsg consumer that listens to activity on the Fedora message bus, and updates +the Product Definition Center database in response. + +%prep +%setup -q -n %{name}-%{version} + +# Remove bundled egg-info in case it exists +rm -rf %{name}.egg-info + +%build +%{__python2} setup.py build + +%install +%{__python2} setup.py install -O1 --skip-build --root=%{buildroot} + +# setuptools installs these, but we don't want them. +rm -rf %{buildroot}%{python2_sitelib}/tests/ + +%check +# The tests require network, but we mock that with vcr +PYTHONPATH=. nosetests -v + +%files +%doc README.rst +%license LICENSE +%{python2_sitelib}/outhouse/ +%{python2_sitelib}/outhouse-%{version}* +#%{_bindir}/pdc-updater-retry + +%changelog diff --git a/outhouse/__init__.py b/outhouse/__init__.py new file mode 100644 index 0000000..dd9c2f6 --- /dev/null +++ b/outhouse/__init__.py @@ -0,0 +1,2 @@ +import outhouse.consumer +import outhouse.handlers diff --git a/outhouse/commands.py b/outhouse/commands.py new file mode 100644 index 0000000..3430c1c --- /dev/null +++ b/outhouse/commands.py @@ -0,0 +1,49 @@ +import logging +import logging.config +import sys + +import fedmsg.config +import fedmsg.encoding + +import outhouse.handlers +import outhouse.utils + +import beanbag.bbexcept + +log = logging.getLogger(__name__) + +# https://github.com/product-definition-center/pdc-client/issues/8 +import pdc_client + + +def retry(): + config = fedmsg.config.load_config() + logging.config.dictConfig(config['logging']) + msg_ids = sys.argv[1:] + if msg_ids: + messages = [outhouse.utils.get_fedmsg(idx) for idx in msg_ids] + else: + log.info("No msg_ids supplied. Reading message payload from stdin.") + messages = [fedmsg.encoding.loads(sys.stdin.read())] + + pdc = pdc_client.PDCClient(**config['outhouse.pdc']) + handlers = outhouse.handlers.load_handlers(config) + for msg in messages: + outhouse.utils.handle_message(pdc, handlers, msg, verbose=True) + + +def initialize(): + config = fedmsg.config.load_config() + logging.config.dictConfig(config['logging']) + pdc = pdc_client.PDCClient(**config['outhouse.pdc']) + pdc.set_comment("Initialized by outhouse") + handlers = outhouse.handlers.load_handlers(config) + for handler in handlers: + log.info("Calling .initialize() on %r" % handler) + pdc.set_comment("Initialized via %r" % handler) + try: + handler.initialize(pdc) + except beanbag.bbexcept.BeanBagException as e: + log.exception(e.response.text) + #raise # TODO - eventually raise here. While in dev, leave it out + log.info("Done initializing.") diff --git a/outhouse/consumer.py b/outhouse/consumer.py new file mode 100644 index 0000000..6a90355 --- /dev/null +++ b/outhouse/consumer.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# This file is part of pdc-updater. +# +# pdc-updater is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# pdc-updater is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with pdc-updater. If not, see . +""" Fedmsg consumer that updates PDC. + +Authors: Ralph Bean + +""" + +import fedmsg.consumers + +import outhouse.handlers + +import logging +log = logging.getLogger(__name__) + + +import pdc_client + + +class OuthouseConsumer(fedmsg.consumers.FedmsgConsumer): + config_key = 'outhouse.enabled' + + def __init__(self, hub): + config = hub.config + + # Ensure this is present, it should be a dict with a url and creds + self.pdc_config = config['outhouse.pdc'] + + # Load all our worker bits. + self.handlers = list(outhouse.handlers.load_handlers(config)) + + # Tell fedmsg to only notify us about topics that our handlers want. + self.topic = [ + '.'.join([config['topic_prefix'], config['environment'], topic]) + for handler in self.handlers for topic in handler.topic_suffixes + ] + + super(OuthouseConsumer, self).__init__(hub) + + def consume(self, msg): + msg = msg['body'] # Remove envelope + idx = msg.get('msg_id', None) + topic = msg.get('topic', None) + self.log.debug("Received %r, %r" % (idx, topic)) + + pdc = pdc_client.PDCClient(**self.pdc_config) + outhouse.utils.handle_message(pdc, self.handlers, msg) diff --git a/outhouse/handlers/__init__.py b/outhouse/handlers/__init__.py new file mode 100644 index 0000000..bfa7f9a --- /dev/null +++ b/outhouse/handlers/__init__.py @@ -0,0 +1,43 @@ +import abc + +import fedmsg.utils + + +def load_handlers(config): + """ Import and instantiate all handlers listed in the given config. """ + for import_path in config['outhouse.handlers']: + cls = fedmsg.utils.load_class(import_path) + handler = cls(config) + yield handler + + +class BaseHandler(object): + """ An abstract base class for handlers to enforce API. """ + __metaclass__ = abc.ABCMeta + + def __init__(self, config): + self.config = config + + @abc.abstractproperty + def topic_suffixes(self): + pass + + @abc.abstractmethod + def can_handle(self, msg): + """ Return True or False if this handler can handle this message. """ + pass + + @abc.abstractmethod + def handle(self, pdc, msg): + """ Handle a fedmsg and update PDC if necessary. """ + pass + + @abc.abstractmethod + def initialize(self, pdc): + """ This needs to be called only once when pdc-updater is first + installed. It should query the original data source and initialize PDC + with a base layer of data. + + It is expected to take a very long time to run. + """ + pass diff --git a/outhouse/handlers/atomic.py b/outhouse/handlers/atomic.py new file mode 100644 index 0000000..e844e89 --- /dev/null +++ b/outhouse/handlers/atomic.py @@ -0,0 +1,66 @@ +import logging +import requests + +import outhouse.handlers +import outhouse.utils + +from pdc_client import get_paged + + +log = logging.getLogger(__name__) + + +class AtomicHandler(outhouse.handlers.BaseHandler): + """ Monitors bodhi updates for packages that affect the Atomic Host. + + """ + # TODO: break out various handlers for different groups + group_type = 'atomic-docker-host' + + def __init__(self, *args, **kwargs): + super(AtomicHandler, self).__init__(*args, **kwargs) + #self.git_url = self.config['.fedora_atomic_git_url'] + + @property + def topic_suffixes(self): + return [ + 'bodhi.update.complete', + ] + + def can_handle(self, msg): + return 'bodhi.update.complete' in msg['topic'] + + def handle(self, pdc, msg): + """ + TODO: + - how do we want to handle batching of re-composing different artifacts + for example, after a push of updates, this will get called with each update + """ + update = msg['msg']['update'] + packages = [] + for build in update['builds']: + packages.append('-'.join(build['nvr'].split('-')[:-2])) + + # Convert 'Fedora 24' to 'fedora-24' + release = update['release']['long_name'].lower().replace(' ', '-') + + # TODO: queue them up for batch processing later + # and we don't want to query pdc for each update fedmsg + + # Query PDC to see if the atomic artifacts are affected by this build + log.info('Querying PDC for artifacts affected by changes in: {}'.format(packages)) + pdc_groups = [ + group for group in get_paged(pdc['component-groups']._) + if group['group_type'] == self.group_type + and group['release'] == release + ] + for group in pdc_groups: + components = set([component['name'] for component in + group['components']]) + affected = set(packages) & components + if affected: + log.info('{} {} affected by {} updates'.format(release, + self.group_type, list(affected))) + + def initialize(self, pdc): + return diff --git a/outhouse/tests/__init__.py b/outhouse/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/outhouse/tests/__init__.py diff --git a/outhouse/tests/handler_tests/__init__.py b/outhouse/tests/handler_tests/__init__.py new file mode 100644 index 0000000..7efe9f0 --- /dev/null +++ b/outhouse/tests/handler_tests/__init__.py @@ -0,0 +1,313 @@ +from os.path import dirname +import functools +import unittest +import logging + +import vcr + +import fedmsg.utils + +from nose.tools import raises + +import outhouse.utils + +import pdc_client.test_helpers + +log = logging.getLogger(__name__) + + +cassette_dir = dirname(dirname(__file__)) + '/vcr-request-data/' + +def mock_404(): + import beanbag.bbexcept + class Mock404Response(object): + status_code = 404 + response = Mock404Response() + raise beanbag.bbexcept.BeanBagException(response, "404, nope.") + +def mock_pdc(function): + @functools.wraps(function) + @pdc_client.test_helpers.mock_api + def wrapper(self, pdc, *args, **kwargs): + # Mock out POST endpoints + pdc.add_endpoint('component-group-types', 'POST', 'wat') + pdc.add_endpoint('component-groups', 'POST', 'wat') + pdc.add_endpoint('global-components', 'POST', 'wat') + pdc.add_endpoint('release-components', 'POST', 'wat') + pdc.add_endpoint('compose-images', 'POST', 'wat') + pdc.add_endpoint('compose-rpms', 'POST', 'wat') + pdc.add_endpoint('persons', 'POST', 'wat') + pdc.add_endpoint('rpms', 'POST', 'wat') + + # Mock out GET endpoints + pdc.add_endpoint('composes/Fedora-24-20151130.n.2', 'GET', mock_404) + + pdc.add_endpoint('releases/fedora-24', 'GET', {}) + pdc.add_endpoint('releases/fedora-23-updates', 'GET', {}) + pdc.add_endpoint('releases/fedora-22-updates', 'GET', {}) + pdc.add_endpoint('releases/fedora-21-updates', 'GET', {}) + pdc.add_endpoint('releases/fedora-20-updates', 'GET', {}) + pdc.add_endpoint('releases/epel-7-updates', 'GET', {}) + pdc.add_endpoint('releases/epel-6-updates', 'GET', {}) + + + pdc.add_endpoint('component-groups', 'GET', { + 'count': 4, + 'next': None, + 'previous': None, + 'results': [{ + 'release': 'fedora-24', + 'description': 'Deps for atomic-docker-host https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + 'group_type': 'atomic-docker-host', + 'id': 1, + }, { + 'release': 'fedora-23-updates', + 'description': 'Deps for atomic-docker-host https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + 'group_type': 'atomic-docker-host', + 'id': 2, + }, { + 'release': 'fedora-22-updates', + 'description': 'Deps for atomic-docker-host https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + 'group_type': 'atomic-docker-host', + 'id': 3, + }, { + 'release': 'fedora-21-updates', + 'description': 'Deps for atomic-docker-host https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + 'group_type': 'atomic-docker-host', + 'id': 3, + }], + }) + + pdc.add_endpoint('persons', 'GET', { + 'count': 2, + 'next': None, + 'previous': None, + 'results': [ + {'username': 'ralph', 'email': 'ralph@fedoraproject.org'}, + {'username': 'lmacken', 'email': 'lmacken@fedoraproject.org'}, + ], + }) + + pdc.add_endpoint('rpms', 'GET', { + 'count': 2, + 'next': None, + 'previous': None, + 'results': [{ + 'name': 'dvisvgm', + 'arch': 'src', + 'epoch': 0, + 'version': '1.11', + 'release': '1.el7', + 'linked_releases': [ + 'epel-7-updates', + ], + 'srpm_name': 'dvisvgm', + 'srpm_nevra': None, + }, { + 'name': 'rubygem-jmespath-doc', + 'arch': 'noarch', + 'epoch': 0, + 'version': '1.1.3', + 'release': '1.el7', + 'linked_releases': [ + 'epel-7-updates', + ], + 'srpm_name': 'rubygem-jmespath', + 'srpm_nevra': 'rubygem-jmespath-1.1.3-1.el7', + }], + }) + + pdc.add_endpoint('release-components', 'GET', { + 'count': 11, + 'next': None, + 'previous': None, + 'results': [ + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'master', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'fedora-24', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'el6', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'epel-6-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'f20', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'fedora-20-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'epel7', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'epel-7-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'f21', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'fedora-21-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'f22', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'fedora-22-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'guake', + 'bugzilla_component': u'guake', + 'dist_git_branch': u'f23', + 'global_component': u'guake', + 'name': u'guake', + 'release': { + 'release_id': u'fedora-23-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'geany', + 'bugzilla_component': u'geany', + 'dist_git_branch': u'master', + 'global_component': u'geany', + 'name': u'geany', + 'release': { + 'release_id': u'fedora-24', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'geany', + 'bugzilla_component': u'geany', + 'dist_git_branch': u'el6', + 'global_component': u'geany', + 'name': u'geany', + 'release': { + 'release_id': u'epel-6-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'geany', + 'bugzilla_component': u'geany', + 'dist_git_branch': u'epel7', + 'global_component': u'geany', + 'name': u'geany', + 'release': { + 'release_id': u'epel-7-updates', + }, + 'type': 'srpm'}, + {'active': True, + 'brew_package': u'geany', + 'bugzilla_component': u'geany', + 'dist_git_branch': u'f23', + 'global_component': u'geany', + 'name': u'geany', + 'release': { + 'release_id': u'fedora-23-updates', + }, + 'type': 'srpm'}, + ] + }) + + pdc.add_endpoint('global-components', 'GET', { + 'count': 11, + 'next': None, + 'previous': None, + 'results': [ + {'name': u'geany'}, + {'name': u'guake'}, + ] + }) + + pdc.add_endpoint('composes', 'GET', { + 'count': 2, + 'next': None, + 'previous': None, + 'results': [{ + "acceptance_testing": "ComposeAcceptanceTestingState.name", + "compose_date": "20151130", + "compose_id": "Fedora-24-20151130.n.2", + "compose_label": "Fedora-24-20151130.n.2", + "compose_respin": "2", + "compose_type": "nightly", + "deleted": "boolean", + "linked_releases": [], + "release": "rawhide", + "rpm_mapping_template": "some url", + "rtt_tested_architectures": { }, + "sigkeys": [], + }], + }) + + return function(self, pdc, *args, **kwargs) + return wrapper + + +class BaseHandlerTest(unittest.TestCase): + maxDiff = None + handler_path = None + config = { + 'outhouse.fas': { + 'base_url': 'whatever', + 'username': 'whatever', + 'password': 'whatever', + }, + 'outhouse.pkgdb_url': 'blihblihblih', + 'outhouse.koji_url': 'http://koji.fedoraproject.org/kojihub', + 'outhouse.old_composes_url': 'https://kojipkgs.fedoraproject.org/compose', + 'outhouse.fedora_atomic_git_url': 'https://git.fedorahosted.org/cgit/fedora-atomic.git/plain/', + } + + def setUp(self): + if not self.handler_path: + log.info("!! Warning - no handler path declared by base class.") + + config = BaseHandlerTest.config + if self.handler_path: + log.info("Initializing handler %s(%r)", self.handler_path, config) + self.handler = fedmsg.utils.load_class(self.handler_path)(config) + + log.info("Setting up vcr cassette in %s", cassette_dir) + filename = cassette_dir + self.id() + self.vcr = vcr.use_cassette(filename, record_mode='new_episodes') + self.vcr.__enter__() + + def tearDown(self): + self.vcr.__exit__() + + +class TestBaseHarness(BaseHandlerTest): + @raises(IOError) + def test_get_nonexistant_fedmsg(self): + outhouse.utils.get_fedmsg('wat') + + def test_get_fedmsg(self): + idx = '2015-6c98c8e3-0dcb-497d-a0d8-0b3d026a4cfb' + msg = outhouse.utils.get_fedmsg(idx) + self.assertEquals(msg['msg']['user']['username'], 'ralph') diff --git a/outhouse/tests/handler_tests/test_atomic.py b/outhouse/tests/handler_tests/test_atomic.py new file mode 100644 index 0000000..c250805 --- /dev/null +++ b/outhouse/tests/handler_tests/test_atomic.py @@ -0,0 +1,49 @@ +import os + +import outhouse.utils +from outhouse.tests.handler_tests import ( + BaseHandlerTest, mock_pdc +) + +here = os.path.dirname(__file__) + +class TestAtomicUpdate(BaseHandlerTest): + handler_path = 'outhouse.handlers.atomic:AtomicComponentGroupHandler' + config = { + } + + def test_cannot_handle_other_git_push(self): + idx = '2016-ab90226c-5fc3-48f8-83de-821e4fe0ade4' + msg = outhouse.utils.get_fedmsg(idx) + result = self.handler.can_handle(msg) + self.assertEquals(result, False) + + def test_can_handle_atomic_update_push(self): + msg = { + "topic": "org.fedoraproject.prod.trac.git.receive", + "msg": { + "commit": { + "branch": "master", + "repo": "fedora-atomic", + }, + }, + } + result = self.handler.can_handle(msg) + self.assertEquals(result, True) + + @mock_pdc + def test_handle_a_push(self, pdc): + msg = { + "topic": "org.fedoraproject.prod.trac.git.receive", + "msg": { + "commit": { + "branch": "master", + "repo": "fedora-atomic", + }, + }, + } + self.handler.handle(pdc, msg) + + # Check number of groups + self.assertEquals(len(pdc.calls['component-group-types']), 4) + self.assertEquals(len(pdc.calls['component-groups']), 8) diff --git a/outhouse/utils.py b/outhouse/utils.py new file mode 100644 index 0000000..8239856 --- /dev/null +++ b/outhouse/utils.py @@ -0,0 +1,94 @@ +import copy +import contextlib + +import requests +import beanbag.bbexcept +import pdc_client + +import logging +log = logging.getLogger(__name__) + +import dogpile.cache +cache = dogpile.cache.make_region() +cache.configure('dogpile.cache.memory', expiration_time=300) + +session = requests.Session() + + +def ensure_component_group_type_exists(pdc, component_group_type): + """ Create a component_group-type in PDC if it doesn't already exist. """ + try: + # Try to create it + pdc['component-group-types']._(dict(name=component_group_type)) + except beanbag.bbexcept.BeanBagException as e: + if e.response.status_code != 400: + raise + body = e.response.json() + if not 'name' in body: + raise + if body['name'] != [u"This field must be unique."]: + raise + + +def get_fedmsg(idx): + url = 'https://apps.fedoraproject.org/datagrepper/id' + response = session.get(url, params=dict(id=idx)) + if not bool(response): + raise IOError("Failed to talk to %r %r" % (response.url, response)) + return response.json() + + +@contextlib.contextmanager +def annotated(client, msg_id): + client.set_comment(msg_id) + try: + yield client + finally: + client.set_comment('No comment.') + + +def handle_message(pdc, handlers, msg, verbose=False): + idx, topic = msg['msg_id'], msg['topic'] + for handler in handlers: + name = type(handler).__name__ + if not handler.can_handle(msg): + if verbose: + log.info("%s could not handle %s" % (name, idx)) + continue + log.info("%s handling %s %s" % (name, idx, topic)) + with annotated(pdc, msg['msg_id']) as client: + try: + handler.handle(client, msg) + except beanbag.bbexcept.BeanBagException as e: + log.error(e.response.text) + raise + + +def tag2release(tag): + if tag == rawhide_tag(): + release = { + 'name': 'Fedora', + 'short': 'fedora', + 'version': tag.strip('f'), + 'release_type': 'ga', + } + release_id = "{short}-{version}".format(**release) + else: + bodhi_info = {r['stable_tag']: r for r in bodhi_releases()}[tag] + if 'EPEL' in bodhi_info['id_prefix']: + release = { + 'name': 'Fedora EPEL', + 'short': 'epel', + 'version': bodhi_info['version'], + 'release_type': 'updates', + } + else: + release = { + 'name': 'Fedora Updates', + 'short': 'fedora', + 'version': bodhi_info['version'], + 'release_type': 'updates', + } + release_id = "{short}-{version}-{release_type}".format(**release) + + return release_id, release diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..537c404 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +from setuptools import setup, find_packages + + +requirements = [ + 'fedmsg', + 'fedmsg[commands]', + 'fedmsg[consumers]', + 'requests', + 'dogpile.cache', + 'python-fedora', + #'packagedb-cli', + 'pdc-client', +] + + +with open('test-requirements.txt', 'r') as f: + test_requirements = f.readlines() + + +setup( + name='outhouse', + version='0.0.1', + description='', + license='GPLv2+', + author='Luke Macken', + author_email='lmacken@redhat.com', + url='https://pagure.io/outhouse', + install_requires=requirements, + tests_require=test_requirements, + packages=find_packages(), + include_data=True, + entry_points=""" + [moksha.consumer] + consumer = outhouse.consumer:OuthouseConsumer + """ + + #[console_scripts] + #pdc-updater-initialize = pdcupdater.commands:initialize + #pdc-updater-audit = pdcupdater.commands:audit + #pdc-updater-retry = pdcupdater.commands:retry +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..2b1cce2 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +nose +vcrpy +mock