From 645b72ce3ec5da0a195eb6352eb1dfb4d2680e64 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 1/54] Initial commit of a Flask API. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 0b3758c..12da9b5 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,69 +1,16 @@ -"""kiskadee API.""" -from flask import Flask, jsonify -from flask import request -from flask_cors import CORS - +from flask import Flask from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version, Analysis -from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ - AnalysisSchema +from kiskadee.model import Package, Fetcher, Version kiskadee = Flask(__name__) - -CORS(kiskadee) +db_session = Database().session -@kiskadee.route('/fetchers') +@kiskadee.route('/') def index(): - """Get the list of available fetchers.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - fetchers = db_session.query(Fetcher).all() - fetcher_schema = FetcherSchema(many=True) - result = fetcher_schema.dump(fetchers) - return jsonify({'fetchers': result.data}) - - -@kiskadee.route('/packages') -def packages(): - """Get the list of analyzed packages.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - packages = db_session.query(Package).all() - package_schema = PackageSchema(many=True) - result = package_schema.dump(packages) - return jsonify({'packages': result.data}) - - -@kiskadee.route('/analysis///') -def package_analysis(pkg_name, version): - """Get the a analysis of some package version.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - package_id = ( - db_session.query(Package) - .filter(Package.name == pkg_name).first().id - ) - version_id = ( - db_session.query(Version) - .filter(Version.number == version) - .filter(Version.package_id == package_id).first().id - ) - analysis = ( - db_session.query(Analysis) - .filter(Analysis.version_id == version_id).first() - ) - - analysis_schema = AnalysisSchema() - result = analysis_schema.dump(analysis) - return jsonify({'analysis': result.data}) - + return db_session.query(Package).first().name -def kiskadee_db_session(): - """Return a kiskadee database session.""" - return Database().session +if __name__ == '__main__': + kiskadee.run(debug=True) -def main(): - """Initialize the kiskadee API.""" - kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index f1af6c5..a33e9a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,3 @@ pydocstyle coverage nose flask -Flask-Restless -marshmallow -flask-cors -coverage -nose From 337e8e7eef1f78c0ebcc89bef29cd5e1dba6fdc8 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 2/54] Add endpoint to get analyzed packages. - Add endpoint to get the available fetchers. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 12da9b5..da0448a 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,16 +1,30 @@ -from flask import Flask +import json +from flask import Flask, jsonify +from flask import request + from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version +from kiskadee.api.serializers import PackageSchema, FetcherSchema kiskadee = Flask(__name__) db_session = Database().session -@kiskadee.route('/') +@kiskadee.route('/fetchers') def index(): - return db_session.query(Package).first().name + if request.method == 'GET': + fetchers = db_session.query(Fetcher).all() + fetcher_schema = FetcherSchema(many=True) + result = fetcher_schema.dump(fetchers) + return jsonify({'fetcher': result.data}) +@kiskadee.route('/packages') +def packages(): + if request.method == 'GET': + packages = db_session.query(Package).all() + package_schema = PackageSchema(many=True) + result = package_schema.dump(packages) + return jsonify({'packages': result.data}) if __name__ == '__main__': kiskadee.run(debug=True) - diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 5b19b8c..9bb9072 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,60 +1,22 @@ -"""Provide objects to serialize the kiskadee models.""" - -from marshmallow import Schema, fields -from kiskadee.model import Package, Fetcher, Analysis, Version - - -class AnalysisSchema(Schema): - """Provide a serializer to the Analysis model.""" - - id = fields.Int() - version_id = fields.Int() - analyzer_id = fields.Int() - raw = fields.Str() - - def make_object(self, data): - """Serialize a Analysis object.""" - print('MAKING OBJECT FROM', data) - return Analysis(**data) - - -class VersionSchema(Schema): - """Provide a serializer to the Package model.""" - - id = fields.Int() - number = fields.Str() - package_id = fields.Int() - analysis = fields.Nested(AnalysisSchema, many=True) - - def make_object(self, data): - """Serialize a Package object.""" - print('MAKING OBJECT FROM', data) - return Version(**data) - +from marshmallow import Schema, fields, ValidationError, pre_load class FetcherSchema(Schema): - """Provide a serializer to the Fetcher model.""" - id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): - """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) - class PackageSchema(Schema): - """Provide a serializer to the Package model.""" - id = fields.Int() name = fields.Str() - fetcher_id = fields.Int() - versions = fields.Nested(VersionSchema, many=True) + target = fields.Str() + fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): - """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) + diff --git a/requirements.txt b/requirements.txt index a33e9a9..55c99e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,5 @@ pydocstyle coverage nose flask +Flask-Restless +marshmallow From 6651df2a7f4c7bf5e082b3fc2e3bd934e12c9de5 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 3/54] Add endpoint to get a package analysis. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index da0448a..c609bcd 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -3,8 +3,9 @@ from flask import Flask, jsonify from flask import request from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version -from kiskadee.api.serializers import PackageSchema, FetcherSchema +from kiskadee.model import Package, Fetcher, Version, Analysis +from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ + AnalysisSchema kiskadee = Flask(__name__) db_session = Database().session @@ -26,5 +27,19 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) +@kiskadee.route('/analysis///') +def package_analysis(pkg_name, version): + if request.method == 'GET': + package = db_session.query(Package)\ + .filter(Package.name == pkg_name).first().id + version = db_session.query(Version)\ + .filter(Version.package_id == package).first().id + analysis = db_session.query(Analysis)\ + .filter(Analysis.version_id == version).first() + + analysis_schema = AnalysisSchema() + result = analysis_schema.dump(analysis) + return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run(debug=True) diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 9bb9072..34f1866 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -20,3 +20,12 @@ class PackageSchema(Schema): print('MAKING OBJECT FROM', data) return Package(**data) +class AnalysisSchema(Schema): + id = fields.Int() + version_id = fields.Int() + analyzer_id = fields.Int() + raw = fields.Str() + + def make_object(self, data): + print('MAKING OBJECT FROM', data) + return Analysis(**data) From 91d7117cd4ba8e8509b6709d230bdbdabefeb724 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 4/54] Update Jenkinsfile. --- diff --git a/Jenkinsfile b/Jenkinsfile index f20c554..86c41d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } From dac1350fc293140a520f5a7930fe673a3d6a01f4 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 5/54] Add cross origin support --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index c609bcd..4dc1889 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,6 +1,7 @@ import json from flask import Flask, jsonify from flask import request +from flask_cors import CORS, cross_origin from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -9,7 +10,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ kiskadee = Flask(__name__) db_session = Database().session - +CORS(kiskadee) @kiskadee.route('/fetchers') def index(): @@ -27,6 +28,7 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) + @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): if request.method == 'GET': @@ -42,4 +44,4 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) if __name__ == '__main__': - kiskadee.run(debug=True) + kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index 55c99e6..c663734 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ nose flask Flask-Restless marshmallow +flask-cors From 5752ced58dbfcb38db256d602fd68be510c1694d Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 6/54] use jenkins user to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 86c41d3..722d986 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,21 @@ pipeline { agent any + environment { + USER = "jenkins" + } + stages { stage('Build') { steps { - sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' + sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' - sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' } } stage('Test') { From 0ef7a0c786fc312fd25b22611973854a1d5ac58f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:51 +0000 Subject: [PATCH 7/54] Change repo owner. --- diff --git a/Jenkinsfile b/Jenkinsfile index 722d986..8452c0b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,11 @@ pipeline { } stages { + stage('change-repo-owner') { + steps { + sh 'chown -R ${USER}.${USER} .' + } + } stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' From 8469270ac535e1902c3226c84a3ffc669297e35c Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 8/54] Activate the virtualenv properly. --- diff --git a/Jenkinsfile b/Jenkinsfile index 8452c0b..971efb6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' + sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { From 7825d00d5a3d4c1f49c8860f6dd16b94becb8b46 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 9/54] Source the virtualenv functions. --- diff --git a/Jenkinsfile b/Jenkinsfile index 971efb6..0eeafb8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' + sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { From 8cc3889329c726a2f74e0bf3e205bec3b5ba9140 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 10/54] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index 0eeafb8..9b11fa9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' + sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { From 2006c3f69865e25fed8e8bdefd22b1029eecff55 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 11/54] User sh to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 9b11fa9..edbd04f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' + sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" } } stage('build-docker-images') { From 69a7e225cbfef5f3f6b4dfde0c9982e2801de650 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 12/54] Use default jenkins user. --- diff --git a/Jenkinsfile b/Jenkinsfile index edbd04f..f815e73 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,6 @@ pipeline { agent any - environment { - USER = "jenkins" - } - stages { stage('change-repo-owner') { steps { @@ -13,14 +9,14 @@ pipeline { } stage('Build') { steps { - sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" + sh 'virtualenv -p /usr/bin/python3 .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { steps { - sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' - sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } stage('Test') { From 1f730202ec6ccc0a5efa3e72608120dee37f7910 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 13/54] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index f815e73..f20c554 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,15 +2,10 @@ pipeline { agent any stages { - stage('change-repo-owner') { - steps { - sh 'chown -R ${USER}.${USER} .' - } - } stage('Build') { steps { sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { From 5162781816930b80f25c881ef9f140eba99076ac Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 14/54] Fix tests --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4dc1889..4a96917 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,7 +1,7 @@ -import json +"""Kiskadee API.""" from flask import Flask, jsonify from flask import request -from flask_cors import CORS, cross_origin +from flask_cors import CORS from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -12,16 +12,20 @@ kiskadee = Flask(__name__) db_session = Database().session CORS(kiskadee) + @kiskadee.route('/fetchers') def index(): + """Get the list of available fetchers.""" if request.method == 'GET': fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) return jsonify({'fetcher': result.data}) + @kiskadee.route('/packages') def packages(): + """Get the list of analyzed packages.""" if request.method == 'GET': packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) @@ -31,17 +35,23 @@ def packages(): @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): + """Get the a analysis of some package version.""" if request.method == 'GET': package = db_session.query(Package)\ .filter(Package.name == pkg_name).first().id - version = db_session.query(Version)\ + version = ( + db_session.query(Version) .filter(Version.package_id == package).first().id - analysis = db_session.query(Analysis)\ + ) + analysis = ( + db_session.query(Analysis) .filter(Analysis.version_id == version).first() + ) analysis_schema = AnalysisSchema() result = analysis_schema.dump(analysis) return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run('0.0.0.0') diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 34f1866..3b04abc 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,31 +1,46 @@ -from marshmallow import Schema, fields, ValidationError, pre_load +"""Provide objects to serialize the kiskadee models.""" + +from marshmallow import Schema, fields +from kiskadee.model import Package, Fetcher, Analysis + class FetcherSchema(Schema): + """Provide a serializer to the Fetcher model.""" + id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): + """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) + class PackageSchema(Schema): + """Provide a serializer to the Package model.""" + id = fields.Int() name = fields.Str() target = fields.Str() fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): + """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) + class AnalysisSchema(Schema): + """Provide a serializer to the Analysis model.""" + id = fields.Int() version_id = fields.Int() analyzer_id = fields.Int() raw = fields.Str() def make_object(self, data): + """Serialize a Analysis object.""" print('MAKING OBJECT FROM', data) return Analysis(**data) From 8ed168aeb7476183f46dfa7e17c3e30028f4f615 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 15/54] Add test coverage. --- diff --git a/requirements.txt b/requirements.txt index c663734..f1af6c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,5 @@ flask Flask-Restless marshmallow flask-cors +coverage +nose From 7e289eaed754ba11daa9334fde5a89d34574ee61 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 16/54] Initial test to kiskadee api. --- diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index fa04b3c..e8fe09f 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,48 +1,29 @@ import json +from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model -import kiskadee -from kiskadee.api.app import kiskadee as kiskadee_api -import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee_api.testing = True + kiskadee.testing = True + self.app = kiskadee.test_client() self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() - self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - fetcher = model.Fetcher( + self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) - self.session.add(fetcher) - self.session.commit() + ) def test_get_fetchers(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/fetchers") - self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) - - def test_get_activated_fetcher(self): - - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - response_as_json = json.loads(response.data.decode("utf-8")) - fetcher_name = response_as_json["fetchers"][0]["name"] - self.assertEqual("kiskadee-fetcher", fetcher_name) + self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) if __name__ == '__main__': From 15b3a9b6b43b4499b351d8d4fc4147aea41f8f69 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 17/54] Save analysis as JSON. - related to #40 - This commit generated the bug related with the issue #44. We need to fix it, to release the 0.3 version. --- diff --git a/kiskadee/converter.py b/kiskadee/converter.py index 12ae044..dd38571 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,8 +7,6 @@ from importlib import import_module import shutil import tempfile import os -import json - from firehose.model import Analysis, to_json @@ -40,7 +38,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return json.dumps(analysis_as_json) + return analysis_as_json def import_firehose_parser(parser): From eb57084965d0cc95ccc4c29029679035323de0eb Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:52 +0000 Subject: [PATCH 18/54] Generate the coverage directly from setup.py. --- diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh new file mode 100755 index 0000000..ef480cc --- /dev/null +++ b/run_tests_and_coverage.sh @@ -0,0 +1,3 @@ +#!/bin/bash +coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test +coverage html From 0da04a8a016c85ff5158a5f1df424ec2cf040f9a Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 19/54] Ignore .eggs file when run the coverage. --- diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh index ef480cc..2634599 100755 --- a/run_tests_and_coverage.sh +++ b/run_tests_and_coverage.sh @@ -1,3 +1,3 @@ #!/bin/bash -coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test +coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*" ./setup.py test coverage html From 08d77a33d0a4eb6beba4b950e31e5a1a694406a9 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 20/54] Make api tests not depend of a postgresql database --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4a96917..6be2a26 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -9,7 +9,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ AnalysisSchema kiskadee = Flask(__name__) -db_session = Database().session + CORS(kiskadee) @@ -17,16 +17,18 @@ CORS(kiskadee) def index(): """Get the list of available fetchers.""" if request.method == 'GET': + db_session = kiskadee_db_session() fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) - return jsonify({'fetcher': result.data}) + return jsonify({'fetchers': result.data}) @kiskadee.route('/packages') def packages(): """Get the list of analyzed packages.""" if request.method == 'GET': + db_session = kiskadee_db_session() packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) result = package_schema.dump(packages) @@ -37,8 +39,11 @@ def packages(): def package_analysis(pkg_name, version): """Get the a analysis of some package version.""" if request.method == 'GET': - package = db_session.query(Package)\ + db_session = kiskadee_db_session() + package = ( + db_session.query(Package) .filter(Package.name == pkg_name).first().id + ) version = ( db_session.query(Version) .filter(Version.package_id == package).first().id @@ -53,5 +58,11 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) -if __name__ == '__main__': +def kiskadee_db_session(): + """Return a kiskadee database session.""" + return Database().session + + +def main(): + """Initialize the kiskadee API.""" kiskadee.run('0.0.0.0') diff --git a/kiskadee/converter.py b/kiskadee/converter.py index dd38571..12ae044 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,6 +7,8 @@ from importlib import import_module import shutil import tempfile import os +import json + from firehose.model import Analysis, to_json @@ -38,7 +40,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return analysis_as_json + return json.dumps(analysis_as_json) def import_firehose_parser(parser): diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index e8fe09f..fa04b3c 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,29 +1,48 @@ import json -from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model +import kiskadee +from kiskadee.api.app import kiskadee as kiskadee_api +import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee.testing = True - self.app = kiskadee.test_client() + kiskadee_api.testing = True self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() + self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - self.fetcher = model.Fetcher( + fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) + ) + self.session.add(fetcher) + self.session.commit() def test_get_fetchers(self): + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session + response = self.app.get("/fetchers") + self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) + + def test_get_activated_fetcher(self): + + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) + response_as_json = json.loads(response.data.decode("utf-8")) + fetcher_name = response_as_json["fetchers"][0]["name"] + self.assertEqual("kiskadee-fetcher", fetcher_name) if __name__ == '__main__': From 71d46ebace7dc419a0b5fad2f354b6673b7f9e94 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 21/54] Update README.md - Add CI link in README.me - Add an architecture section to the docs. --- diff --git a/doc/architecture.rst b/doc/architecture.rst index 5d0588d..579cc48 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -25,4 +25,4 @@ kiskadee authors. .. -*Figure One: kiskadee architecture.* +*Figure One: Kiskadee architecture.* From 531e939209fbc3011343aa93beae83111703f3cc Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 22/54] Review README.md --- diff --git a/README.md b/README.md index b030bc2..934d837 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ into a Firehose database. ## Setup ### Dependencies +<<<<<<< HEAD The name of the dependencies are compatible with the Fedora distribution. If you use another operational system, @@ -35,6 +36,32 @@ for our python dependencies. virtualenv -p /usr/bin/python3 . source bin/activate +======= + +Install some package dependencies. The name of the dependencies are compatible +with the Fedora distribution. If you use another operational system, +you will have to find the compatible name for the dependencies. +The `redhat-rpm-config` +package, is a specific Fedora dependency. If you not use Fedora (or a +Red Hat distribution), maybe you will not have to install it. + + - openssl-devel + - python3-devel + - gcc + - redhat-rpm-config python-pip + +### Virtual Environment + +Create a virtualenv to kiskadee. `dnf` is a package manager for the Fedora +distribution, if you not use Fedora, use your package manage to install the +virtualenv and pip packages. The virtualenv package will create a isolate +environment for our python dependencies. + + sudo pip install virtualenv + virtualenv -p /usr/bin/python3 . + source bin/activate + +>>>>>>> 3bc1617... Review README.md Install the python dependencies using pip pip install -e . @@ -47,6 +74,7 @@ To run the static analyzers, you must have If you have configured the Docker engineer properly, run the *docker_build.sh* script. It will build the images for you. +<<<<<<< HEAD chmod u+x docker_build.sh ./docker_build.sh @@ -55,6 +83,15 @@ Now we will create the kiskadee database. You will need to install the postgresql packages for your system. If you use Fedora, follow the next steps, if not, you will have to find out how install postgresql on your system. +======= +chmod u+x docker\_build.sh + +./docker\_build.sh + +### Database +Now we will create the kiskadee database. You will need to install the +postgresql packages for your system. +>>>>>>> 3bc1617... Review README.md sudo dnf install postgresql-server postgresql-contrib sudo systemctl enable postgresql @@ -63,8 +100,15 @@ system. To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). +<<<<<<< HEAD With postgresql installed, you will need to create the kiskadee role and database. +======= +With postgresql installed, you will need to create the kiskadee role. This +role will be used to log in on the database: + +Now create the database: +>>>>>>> 3bc1617... Review README.md sudo su - postgres createdb kiskadee @@ -72,7 +116,8 @@ database. # use kiskadee as password. psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" # go back to your user (ctrl+d) - echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass +<<<<<<< HEAD + echo "localhost:5432:kiskadee:kiskadee:" > ~/.pgpass chmod 600 ~/.pgpass Restart the postgresql service: @@ -86,6 +131,13 @@ Test the database connection: If you was not able to log in on the database, you will need to edit the *pg_hba.conf* and change some rules defined by the postgresql package. On Linux systems this file normally stays at the +======= + echo "localhost:5432:kiskadee:kiskadee:" > ~/.pgpass + chmod 600 ~/.pgpass + +You will need to edit the *pg_hba.conf* to permits the kiskadee user to login +on the database. On Linux systems this file normally stays at the +>>>>>>> 3bc1617... Review README.md `/var/lib/pgsql/data/`. Open this file and change: # "local" is for Unix domain socket connections only @@ -116,21 +168,45 @@ Test the database connection: If you was able to get into the psql shell, the database is properly configured. Leave the shell with ctrl+d. +<<<<<<< HEAD ### Running our first analysis kiskadee reads environment variables from the `util/kiskadee.conf` file. If everything goes well till now, open the *kiskadee.conf* file, and set as +======= +### Running + +Kiskadee read environment variables from the `util/kiskadee.conf` file. +If everything goes well till now, open the kiskadee.conf file, and set as +>>>>>>> 3bc1617... Review README.md active (`active = yes`) only the *example_fetcher*, the other fetchers will stay as `active = no`. Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a +<<<<<<< HEAD example source code. This code is in the *kiskadee/tests/test_source/* directory. kiskadee will decompress the example source, and run the analyzers defined on the *kiskadee.conf* file. You can use any postgresql client to access the database that you have created, and check the analysis maded by kiskadee. +======= +example source code. This code is in the kiskadee/tests/test\_source/ directory. + +## Fetchers + +### Debian Fetcher +If you intend to use the debian fetcher, you will have to install the +`devscripts` package, in order use the necessary Debian tools to run the +fetcher. + + +### Anitya Fetcher +If you intend to run the anitya fetcher, you will have to install fedmsg-hub, +in order to kiskadee be able to consume the fedmsg events. +To install fedmsg-hub follow this steps inside the kiskadee root path: +>>>>>>> 3bc1617... Review README.md ### Running API @@ -145,8 +221,11 @@ To check kiskadee tests and coverage just run: chmod u+x run_tests_and_coverage.sh ./run_tests_and_coverage.sh +<<<<<<< HEAD To check kiskadee coverage open the file *covhtml/index.html*. +======= +>>>>>>> 3bc1617... Review README.md ## Repositories kiskadee daemon and API development are hosted at [pagure](https://pagure.io/kiskadee). From 085c6a55d4566830ef6cd8c0a490411ffa0cc85e Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 23/54] Add instructions to API on README. --- diff --git a/README.md b/README.md index 934d837..b5613b3 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,12 @@ To run the kiskadee api just execute the command: kiskadee_api +### Running API + +To run the kiskadee api just execute the command: + + kiskadee_api + ## Tests and coverage To check kiskadee tests and coverage just run: From bf5b3800234637d346c0872424bbb1473284f5fa Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 24/54] Add instruction to use kiskadee as passwd --- diff --git a/README.md b/README.md index b5613b3..b030bc2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,6 @@ into a Firehose database. ## Setup ### Dependencies -<<<<<<< HEAD The name of the dependencies are compatible with the Fedora distribution. If you use another operational system, @@ -36,32 +35,6 @@ for our python dependencies. virtualenv -p /usr/bin/python3 . source bin/activate -======= - -Install some package dependencies. The name of the dependencies are compatible -with the Fedora distribution. If you use another operational system, -you will have to find the compatible name for the dependencies. -The `redhat-rpm-config` -package, is a specific Fedora dependency. If you not use Fedora (or a -Red Hat distribution), maybe you will not have to install it. - - - openssl-devel - - python3-devel - - gcc - - redhat-rpm-config python-pip - -### Virtual Environment - -Create a virtualenv to kiskadee. `dnf` is a package manager for the Fedora -distribution, if you not use Fedora, use your package manage to install the -virtualenv and pip packages. The virtualenv package will create a isolate -environment for our python dependencies. - - sudo pip install virtualenv - virtualenv -p /usr/bin/python3 . - source bin/activate - ->>>>>>> 3bc1617... Review README.md Install the python dependencies using pip pip install -e . @@ -74,7 +47,6 @@ To run the static analyzers, you must have If you have configured the Docker engineer properly, run the *docker_build.sh* script. It will build the images for you. -<<<<<<< HEAD chmod u+x docker_build.sh ./docker_build.sh @@ -83,15 +55,6 @@ Now we will create the kiskadee database. You will need to install the postgresql packages for your system. If you use Fedora, follow the next steps, if not, you will have to find out how install postgresql on your system. -======= -chmod u+x docker\_build.sh - -./docker\_build.sh - -### Database -Now we will create the kiskadee database. You will need to install the -postgresql packages for your system. ->>>>>>> 3bc1617... Review README.md sudo dnf install postgresql-server postgresql-contrib sudo systemctl enable postgresql @@ -100,15 +63,8 @@ postgresql packages for your system. To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). -<<<<<<< HEAD With postgresql installed, you will need to create the kiskadee role and database. -======= -With postgresql installed, you will need to create the kiskadee role. This -role will be used to log in on the database: - -Now create the database: ->>>>>>> 3bc1617... Review README.md sudo su - postgres createdb kiskadee @@ -116,8 +72,7 @@ Now create the database: # use kiskadee as password. psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" # go back to your user (ctrl+d) -<<<<<<< HEAD - echo "localhost:5432:kiskadee:kiskadee:" > ~/.pgpass + echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass chmod 600 ~/.pgpass Restart the postgresql service: @@ -131,13 +86,6 @@ Test the database connection: If you was not able to log in on the database, you will need to edit the *pg_hba.conf* and change some rules defined by the postgresql package. On Linux systems this file normally stays at the -======= - echo "localhost:5432:kiskadee:kiskadee:" > ~/.pgpass - chmod 600 ~/.pgpass - -You will need to edit the *pg_hba.conf* to permits the kiskadee user to login -on the database. On Linux systems this file normally stays at the ->>>>>>> 3bc1617... Review README.md `/var/lib/pgsql/data/`. Open this file and change: # "local" is for Unix domain socket connections only @@ -168,51 +116,21 @@ Test the database connection: If you was able to get into the psql shell, the database is properly configured. Leave the shell with ctrl+d. -<<<<<<< HEAD ### Running our first analysis kiskadee reads environment variables from the `util/kiskadee.conf` file. If everything goes well till now, open the *kiskadee.conf* file, and set as -======= -### Running - -Kiskadee read environment variables from the `util/kiskadee.conf` file. -If everything goes well till now, open the kiskadee.conf file, and set as ->>>>>>> 3bc1617... Review README.md active (`active = yes`) only the *example_fetcher*, the other fetchers will stay as `active = no`. Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a -<<<<<<< HEAD example source code. This code is in the *kiskadee/tests/test_source/* directory. kiskadee will decompress the example source, and run the analyzers defined on the *kiskadee.conf* file. You can use any postgresql client to access the database that you have created, and check the analysis maded by kiskadee. -======= -example source code. This code is in the kiskadee/tests/test\_source/ directory. - -## Fetchers - -### Debian Fetcher -If you intend to use the debian fetcher, you will have to install the -`devscripts` package, in order use the necessary Debian tools to run the -fetcher. - - -### Anitya Fetcher -If you intend to run the anitya fetcher, you will have to install fedmsg-hub, -in order to kiskadee be able to consume the fedmsg events. -To install fedmsg-hub follow this steps inside the kiskadee root path: ->>>>>>> 3bc1617... Review README.md - -### Running API - -To run the kiskadee api just execute the command: - - kiskadee_api ### Running API @@ -227,11 +145,8 @@ To check kiskadee tests and coverage just run: chmod u+x run_tests_and_coverage.sh ./run_tests_and_coverage.sh -<<<<<<< HEAD To check kiskadee coverage open the file *covhtml/index.html*. -======= ->>>>>>> 3bc1617... Review README.md ## Repositories kiskadee daemon and API development are hosted at [pagure](https://pagure.io/kiskadee). From 6861d91d2f84ac770bc887e33db60170d4be3f5f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 25/54] Initial commit of a Flask API. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 6be2a26..12da9b5 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,68 +1,16 @@ -"""Kiskadee API.""" -from flask import Flask, jsonify -from flask import request -from flask_cors import CORS - +from flask import Flask from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version, Analysis -from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ - AnalysisSchema +from kiskadee.model import Package, Fetcher, Version kiskadee = Flask(__name__) - -CORS(kiskadee) +db_session = Database().session -@kiskadee.route('/fetchers') +@kiskadee.route('/') def index(): - """Get the list of available fetchers.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - fetchers = db_session.query(Fetcher).all() - fetcher_schema = FetcherSchema(many=True) - result = fetcher_schema.dump(fetchers) - return jsonify({'fetchers': result.data}) - - -@kiskadee.route('/packages') -def packages(): - """Get the list of analyzed packages.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - packages = db_session.query(Package).all() - package_schema = PackageSchema(many=True) - result = package_schema.dump(packages) - return jsonify({'packages': result.data}) - - -@kiskadee.route('/analysis///') -def package_analysis(pkg_name, version): - """Get the a analysis of some package version.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - package = ( - db_session.query(Package) - .filter(Package.name == pkg_name).first().id - ) - version = ( - db_session.query(Version) - .filter(Version.package_id == package).first().id - ) - analysis = ( - db_session.query(Analysis) - .filter(Analysis.version_id == version).first() - ) - - analysis_schema = AnalysisSchema() - result = analysis_schema.dump(analysis) - return jsonify({'analysis': result.data}) - + return db_session.query(Package).first().name -def kiskadee_db_session(): - """Return a kiskadee database session.""" - return Database().session +if __name__ == '__main__': + kiskadee.run(debug=True) -def main(): - """Initialize the kiskadee API.""" - kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index f1af6c5..953963a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,11 +12,4 @@ packaging pyyaml flake8 pydocstyle -coverage -nose flask -Flask-Restless -marshmallow -flask-cors -coverage -nose From 23e2a45bdad48990df08b204316bc67ad0d4916d Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 26/54] Add endpoint to get analyzed packages. - Add endpoint to get the available fetchers. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 12da9b5..da0448a 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,16 +1,30 @@ -from flask import Flask +import json +from flask import Flask, jsonify +from flask import request + from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version +from kiskadee.api.serializers import PackageSchema, FetcherSchema kiskadee = Flask(__name__) db_session = Database().session -@kiskadee.route('/') +@kiskadee.route('/fetchers') def index(): - return db_session.query(Package).first().name + if request.method == 'GET': + fetchers = db_session.query(Fetcher).all() + fetcher_schema = FetcherSchema(many=True) + result = fetcher_schema.dump(fetchers) + return jsonify({'fetcher': result.data}) +@kiskadee.route('/packages') +def packages(): + if request.method == 'GET': + packages = db_session.query(Package).all() + package_schema = PackageSchema(many=True) + result = package_schema.dump(packages) + return jsonify({'packages': result.data}) if __name__ == '__main__': kiskadee.run(debug=True) - diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 3b04abc..9bb9072 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,46 +1,22 @@ -"""Provide objects to serialize the kiskadee models.""" - -from marshmallow import Schema, fields -from kiskadee.model import Package, Fetcher, Analysis - +from marshmallow import Schema, fields, ValidationError, pre_load class FetcherSchema(Schema): - """Provide a serializer to the Fetcher model.""" - id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): - """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) - class PackageSchema(Schema): - """Provide a serializer to the Package model.""" - id = fields.Int() name = fields.Str() target = fields.Str() fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): - """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) - -class AnalysisSchema(Schema): - """Provide a serializer to the Analysis model.""" - - id = fields.Int() - version_id = fields.Int() - analyzer_id = fields.Int() - raw = fields.Str() - - def make_object(self, data): - """Serialize a Analysis object.""" - print('MAKING OBJECT FROM', data) - return Analysis(**data) diff --git a/requirements.txt b/requirements.txt index 953963a..fbff5eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ pyyaml flake8 pydocstyle flask +Flask-Restless +marshmallow From 77370314e053b67c5a6db1b3e789a428785fbf83 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:53 +0000 Subject: [PATCH 27/54] Add endpoint to get a package analysis. --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index da0448a..c609bcd 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -3,8 +3,9 @@ from flask import Flask, jsonify from flask import request from kiskadee.database import Database -from kiskadee.model import Package, Fetcher, Version -from kiskadee.api.serializers import PackageSchema, FetcherSchema +from kiskadee.model import Package, Fetcher, Version, Analysis +from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ + AnalysisSchema kiskadee = Flask(__name__) db_session = Database().session @@ -26,5 +27,19 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) +@kiskadee.route('/analysis///') +def package_analysis(pkg_name, version): + if request.method == 'GET': + package = db_session.query(Package)\ + .filter(Package.name == pkg_name).first().id + version = db_session.query(Version)\ + .filter(Version.package_id == package).first().id + analysis = db_session.query(Analysis)\ + .filter(Analysis.version_id == version).first() + + analysis_schema = AnalysisSchema() + result = analysis_schema.dump(analysis) + return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run(debug=True) diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 9bb9072..34f1866 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -20,3 +20,12 @@ class PackageSchema(Schema): print('MAKING OBJECT FROM', data) return Package(**data) +class AnalysisSchema(Schema): + id = fields.Int() + version_id = fields.Int() + analyzer_id = fields.Int() + raw = fields.Str() + + def make_object(self, data): + print('MAKING OBJECT FROM', data) + return Analysis(**data) From 7da0bc26292503dc43b65d0f27e1de5ee18a65f9 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 28/54] Update Jenkinsfile. --- diff --git a/Jenkinsfile b/Jenkinsfile index f20c554..86c41d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } From 579e9c67c4ff33a3a230eee5d72f62d02c83f6bd Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 29/54] Add cross origin support --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index c609bcd..4dc1889 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,6 +1,7 @@ import json from flask import Flask, jsonify from flask import request +from flask_cors import CORS, cross_origin from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -9,7 +10,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ kiskadee = Flask(__name__) db_session = Database().session - +CORS(kiskadee) @kiskadee.route('/fetchers') def index(): @@ -27,6 +28,7 @@ def packages(): result = package_schema.dump(packages) return jsonify({'packages': result.data}) + @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): if request.method == 'GET': @@ -42,4 +44,4 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) if __name__ == '__main__': - kiskadee.run(debug=True) + kiskadee.run('0.0.0.0') diff --git a/requirements.txt b/requirements.txt index fbff5eb..1ba4deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ pydocstyle flask Flask-Restless marshmallow +flask-cors From cf65d9cdd8f0ecfa31d0a3d3c0e153180d1cb49b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 30/54] use jenkins user to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 86c41d3..c13f226 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,37 +1,26 @@ pipeline { agent any + environment { + USER = "jenkins" + } + stages { stage('Build') { steps { - sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' + sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { steps { - sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' - sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' + sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' } } stage('Test') { steps { - sh "chmod u+x run_tests_and_coverage.sh" - sh "source bin/activate && ./run_tests_and_coverage.sh" - } - - post { - success { - // publish html - publishHTML target: [ - allowMissing: false, - alwaysLinkToLastBuild: false, - keepAll: true, - reportDir: 'htmlcov', - reportFiles: 'index.html', - reportName: 'coverage report' - ] - } + sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} python setup.py test' } } } From a71b7392ff9df54e18de4b3f9da496734f1f5a2c Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 31/54] Change repo owner. --- diff --git a/Jenkinsfile b/Jenkinsfile index c13f226..3b6f99b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,11 @@ pipeline { } stages { + stage('change-repo-owner') { + steps { + sh 'chown -R ${USER}.${USER} .' + } + } stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' From 99bf0491f69cb4405c6979315d2714899f4ff8f9 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 32/54] Activate the virtualenv properly. --- diff --git a/Jenkinsfile b/Jenkinsfile index 3b6f99b..101fd9f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} pip install -e .' + sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sh 'sudo -H -u ${USER} source bin/activate && sudo -H -u ${USER} python setup.py test' + sh 'source bin/activate && sudo -H -u ${USER} python setup.py test' } } } From 74ecff133fc43b35aa8545ec0ffac352a90ff56d Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 33/54] Source the virtualenv functions. --- diff --git a/Jenkinsfile b/Jenkinsfile index 101fd9f..e7ca1e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && sudo -H -u ${USER} pip install -e .' + sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sh 'source bin/activate && sudo -H -u ${USER} python setup.py test' + sudo -H -u ${USER} sh -c 'source bin/activate && python setup.py test' } } } From f587b4d0a59f88ae9bf5a149166061495b5c8d83 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 34/54] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index e7ca1e5..0224166 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u ${USER} sh -c 'source bin/activate && pip install -e .' + sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sudo -H -u ${USER} sh -c 'source bin/activate && python setup.py test' + sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test' } } } From cbbcef1967e9b22ce4beabf0f9b575c9cda02914 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 35/54] User sh to run commands. --- diff --git a/Jenkinsfile b/Jenkinsfile index 0224166..76b9d45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { stage('Build') { steps { sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .' + sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" } } stage('build-docker-images') { @@ -25,7 +25,7 @@ pipeline { } stage('Test') { steps { - sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test' + sh "sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test'" } } } From d25405a6bc874bfee53699d8b9ac33cb4b318d0b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 36/54] Use default jenkins user. --- diff --git a/Jenkinsfile b/Jenkinsfile index 76b9d45..25bd3c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,6 @@ pipeline { agent any - environment { - USER = "jenkins" - } - stages { stage('change-repo-owner') { steps { @@ -13,19 +9,20 @@ pipeline { } stage('Build') { steps { - sh 'sudo -H -u ${USER} virtualenv -p /usr/bin/python3 .' - sh "sudo -H -u jenkins sh -c 'source bin/activate && pip install -e .'" + sh 'virtualenv -p /usr/bin/python3 .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { steps { - sh '(sudo -H -u ${USER} cd util/dockerfiles/cppcheck && sudo -H -u ${USER} docker build . -t cppcheck)' - sh '(sudo -H -u ${USER} cd util/dockerfiles/flawfinder && sudo -H -u ${USER} docker build . -t flawfinder)' + sh '(cd util/dockerfiles/cppcheck && docker build . -t cppcheck)' + sh '(cd util/dockerfiles/flawfinder && docker build . -t flawfinder)' } } stage('Test') { steps { - sh "sudo -H -u jenkins sh -c 'source bin/activate && python setup.py test'" + sh "echo $UID && echo $USER" + sh "source bin/activate && python setup.py test" } } } From 0549fcaf43e815552cc5e8a4573401382aa51272 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:54 +0000 Subject: [PATCH 37/54] Fix typo. --- diff --git a/Jenkinsfile b/Jenkinsfile index 25bd3c3..2fe8af0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,15 +2,10 @@ pipeline { agent any stages { - stage('change-repo-owner') { - steps { - sh 'chown -R ${USER}.${USER} .' - } - } stage('Build') { steps { sh 'virtualenv -p /usr/bin/python3 .' - sh 'source bin/activate && pip install -e .' + sh 'source bin/activate && pip install -e .' } } stage('build-docker-images') { From fc0bc94179cb4650f71292eed5b065745d14b33f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 38/54] Remove unused code from Jenkinsfile. --- diff --git a/Jenkinsfile b/Jenkinsfile index 2fe8af0..a311405 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,6 @@ pipeline { } stage('Test') { steps { - sh "echo $UID && echo $USER" sh "source bin/activate && python setup.py test" } } From 73d7c32913fe54020bf2bb9404839cc592e88d32 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 39/54] Fix tests --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4dc1889..4a96917 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -1,7 +1,7 @@ -import json +"""Kiskadee API.""" from flask import Flask, jsonify from flask import request -from flask_cors import CORS, cross_origin +from flask_cors import CORS from kiskadee.database import Database from kiskadee.model import Package, Fetcher, Version, Analysis @@ -12,16 +12,20 @@ kiskadee = Flask(__name__) db_session = Database().session CORS(kiskadee) + @kiskadee.route('/fetchers') def index(): + """Get the list of available fetchers.""" if request.method == 'GET': fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) return jsonify({'fetcher': result.data}) + @kiskadee.route('/packages') def packages(): + """Get the list of analyzed packages.""" if request.method == 'GET': packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) @@ -31,17 +35,23 @@ def packages(): @kiskadee.route('/analysis///') def package_analysis(pkg_name, version): + """Get the a analysis of some package version.""" if request.method == 'GET': package = db_session.query(Package)\ .filter(Package.name == pkg_name).first().id - version = db_session.query(Version)\ + version = ( + db_session.query(Version) .filter(Version.package_id == package).first().id - analysis = db_session.query(Analysis)\ + ) + analysis = ( + db_session.query(Analysis) .filter(Analysis.version_id == version).first() + ) analysis_schema = AnalysisSchema() result = analysis_schema.dump(analysis) return jsonify({'analysis': result.data}) + if __name__ == '__main__': kiskadee.run('0.0.0.0') diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 34f1866..3b04abc 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,31 +1,46 @@ -from marshmallow import Schema, fields, ValidationError, pre_load +"""Provide objects to serialize the kiskadee models.""" + +from marshmallow import Schema, fields +from kiskadee.model import Package, Fetcher, Analysis + class FetcherSchema(Schema): + """Provide a serializer to the Fetcher model.""" + id = fields.Int() name = fields.Str() target = fields.Str() description = fields.Str() def make_object(self, data): + """Serialize a Fetcher object.""" print('MAKING OBJECT FROM', data) return Fetcher(**data) + class PackageSchema(Schema): + """Provide a serializer to the Package model.""" + id = fields.Int() name = fields.Str() target = fields.Str() fetcher_id = fields.Nested(FetcherSchema) def make_object(self, data): + """Serialize a Package object.""" print('MAKING OBJECT FROM', data) return Package(**data) + class AnalysisSchema(Schema): + """Provide a serializer to the Analysis model.""" + id = fields.Int() version_id = fields.Int() analyzer_id = fields.Int() raw = fields.Str() def make_object(self, data): + """Serialize a Analysis object.""" print('MAKING OBJECT FROM', data) return Analysis(**data) From eaf4c19d668f870d6c53a85a51b94ee60dac4704 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 40/54] Add test coverage. --- diff --git a/Jenkinsfile b/Jenkinsfile index a311405..ce64eb7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,7 @@ pipeline { } stage('Test') { steps { - sh "source bin/activate && python setup.py test" + sh "source bin/activate && python kiskadee_coverage.py" } } } diff --git a/kiskadee_coverage.py b/kiskadee_coverage.py index b8014df..80894ca 100644 --- a/kiskadee_coverage.py +++ b/kiskadee_coverage.py @@ -6,8 +6,7 @@ sources = [ 'kiskadee.queue', 'kiskadee.runner', 'kiskadee.model', - 'kiskadee.util', - 'kiskadee.api.app' + 'kiskadee.util' ] cov = Coverage(source=sources, omit="lib/*") diff --git a/requirements.txt b/requirements.txt index 1ba4deb..312cf76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,5 @@ flask Flask-Restless marshmallow flask-cors +coverage +nose From c8fcb65a4ad966f12a81297ea1ac134f3e78be90 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 41/54] Publish the html report. --- diff --git a/Jenkinsfile b/Jenkinsfile index ce64eb7..a7d4b9a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,6 +18,20 @@ pipeline { steps { sh "source bin/activate && python kiskadee_coverage.py" } + + post { + success { + // publish html + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'covhtml', + reportFiles: 'index.html', + reportName: 'coverage report' + ] + } + } } } } From e58f8668927158b8113b04a1a17f33c10327c2fe Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 42/54] Initial test to kiskadee api. --- diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index fa04b3c..e8fe09f 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,48 +1,29 @@ import json +from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model -import kiskadee -from kiskadee.api.app import kiskadee as kiskadee_api -import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee_api.testing = True + kiskadee.testing = True + self.app = kiskadee.test_client() self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() - self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - fetcher = model.Fetcher( + self.fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) - self.session.add(fetcher) - self.session.commit() + ) def test_get_fetchers(self): - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session - response = self.app.get("/fetchers") - self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) - - def test_get_activated_fetcher(self): - - def mock_kiskadee_db_session(): - return self.session - - kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - response_as_json = json.loads(response.data.decode("utf-8")) - fetcher_name = response_as_json["fetchers"][0]["name"] - self.assertEqual("kiskadee-fetcher", fetcher_name) + self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) if __name__ == '__main__': diff --git a/kiskadee_coverage.py b/kiskadee_coverage.py index 80894ca..b8014df 100644 --- a/kiskadee_coverage.py +++ b/kiskadee_coverage.py @@ -6,7 +6,8 @@ sources = [ 'kiskadee.queue', 'kiskadee.runner', 'kiskadee.model', - 'kiskadee.util' + 'kiskadee.util', + 'kiskadee.api.app' ] cov = Coverage(source=sources, omit="lib/*") From 572bd70cf34208ef97807a909c5d7b52dc21b578 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 43/54] Save analysis as JSON. - related to #40 - This commit generated the bug related with the issue #44. We need to fix it, to release the 0.3 version. --- diff --git a/kiskadee/converter.py b/kiskadee/converter.py index 12ae044..dd38571 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,8 +7,6 @@ from importlib import import_module import shutil import tempfile import os -import json - from firehose.model import Analysis, to_json @@ -40,7 +38,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return json.dumps(analysis_as_json) + return analysis_as_json def import_firehose_parser(parser): From c065aa8e4831a14654afbd875236b25d0b6ef6a6 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 44/54] Generate the coverage directly from setup.py. --- diff --git a/Jenkinsfile b/Jenkinsfile index a7d4b9a..f20c554 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,7 +16,8 @@ pipeline { } stage('Test') { steps { - sh "source bin/activate && python kiskadee_coverage.py" + sh "chmod u+x run_tests_and_coverage.sh" + sh "source bin/activate && ./run_tests_and_coverage.sh" } post { @@ -26,7 +27,7 @@ pipeline { allowMissing: false, alwaysLinkToLastBuild: false, keepAll: true, - reportDir: 'covhtml', + reportDir: 'htmlcov', reportFiles: 'index.html', reportName: 'coverage report' ] diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh index 2634599..ef480cc 100755 --- a/run_tests_and_coverage.sh +++ b/run_tests_and_coverage.sh @@ -1,3 +1,3 @@ #!/bin/bash -coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*" ./setup.py test +coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test coverage html From 57d5ffa92f4f3949cdc573af26e522f7033e77e6 Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 45/54] Ignore .eggs file when run the coverage. --- diff --git a/run_tests_and_coverage.sh b/run_tests_and_coverage.sh index ef480cc..2634599 100755 --- a/run_tests_and_coverage.sh +++ b/run_tests_and_coverage.sh @@ -1,3 +1,3 @@ #!/bin/bash -coverage run --omit="lib/*","setup.py","kiskadee/tests/*" ./setup.py test +coverage run --omit="lib/*","setup.py","kiskadee/tests/*",".eggs/*" ./setup.py test coverage html From 99d9b5a4da61903be567433e616d1152c8a74dcc Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 46/54] Add an architecture section to the docs. --- diff --git a/doc/architecture.rst b/doc/architecture.rst index 579cc48..7af09aa 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -26,3 +26,4 @@ kiskadee authors. .. *Figure One: Kiskadee architecture.* + From 2b07c5b1ad0aa1fc921b9b750a869c13afae299b Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:55 +0000 Subject: [PATCH 47/54] Make api tests not depend of a postgresql database --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4a96917..6be2a26 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -9,7 +9,7 @@ from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ AnalysisSchema kiskadee = Flask(__name__) -db_session = Database().session + CORS(kiskadee) @@ -17,16 +17,18 @@ CORS(kiskadee) def index(): """Get the list of available fetchers.""" if request.method == 'GET': + db_session = kiskadee_db_session() fetchers = db_session.query(Fetcher).all() fetcher_schema = FetcherSchema(many=True) result = fetcher_schema.dump(fetchers) - return jsonify({'fetcher': result.data}) + return jsonify({'fetchers': result.data}) @kiskadee.route('/packages') def packages(): """Get the list of analyzed packages.""" if request.method == 'GET': + db_session = kiskadee_db_session() packages = db_session.query(Package).all() package_schema = PackageSchema(many=True) result = package_schema.dump(packages) @@ -37,8 +39,11 @@ def packages(): def package_analysis(pkg_name, version): """Get the a analysis of some package version.""" if request.method == 'GET': - package = db_session.query(Package)\ + db_session = kiskadee_db_session() + package = ( + db_session.query(Package) .filter(Package.name == pkg_name).first().id + ) version = ( db_session.query(Version) .filter(Version.package_id == package).first().id @@ -53,5 +58,11 @@ def package_analysis(pkg_name, version): return jsonify({'analysis': result.data}) -if __name__ == '__main__': +def kiskadee_db_session(): + """Return a kiskadee database session.""" + return Database().session + + +def main(): + """Initialize the kiskadee API.""" kiskadee.run('0.0.0.0') diff --git a/kiskadee/converter.py b/kiskadee/converter.py index dd38571..12ae044 100644 --- a/kiskadee/converter.py +++ b/kiskadee/converter.py @@ -7,6 +7,8 @@ from importlib import import_module import shutil import tempfile import os +import json + from firehose.model import Analysis, to_json @@ -38,7 +40,7 @@ def to_firehose(bytes_input, analyzer): analysis_as_json = to_json(Analysis.from_xml(f)) shutil.rmtree(tempdir) - return analysis_as_json + return json.dumps(analysis_as_json) def import_firehose_parser(parser): diff --git a/kiskadee/runner.py b/kiskadee/runner.py index f3b6b80..e115353 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -131,10 +131,10 @@ class Runner: ) ) uncompressed_source_path = tempfile.mkdtemp() - try: - shutil.unpack_archive( - compressed_source, - uncompressed_source_path + shutil.unpack_archive(compressed_source, uncompressed_source_path) + kiskadee.logger.debug( + 'ANALYSIS: Unpacking {} source in {} path' + .format(package['name'], uncompressed_source_path) ) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index e8fe09f..fa04b3c 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -1,29 +1,48 @@ import json -from kiskadee.api.app import kiskadee import unittest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import kiskadee.model as model +import kiskadee +from kiskadee.api.app import kiskadee as kiskadee_api +import kiskadee.api.app class ApiTestCase(unittest.TestCase): def setUp(self): - kiskadee.testing = True - self.app = kiskadee.test_client() + kiskadee_api.testing = True self.engine = create_engine('sqlite:///:memory:') Session = sessionmaker(bind=self.engine) self.session = Session() + self.app = kiskadee_api.test_client() model.Base.metadata.create_all(self.engine) model.create_analyzers(self.session) - self.fetcher = model.Fetcher( + fetcher = model.Fetcher( name='kiskadee-fetcher', target='university' - ) + ) + self.session.add(fetcher) + self.session.commit() def test_get_fetchers(self): + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session + response = self.app.get("/fetchers") + self.assertIn("fetchers", json.loads(response.data.decode("utf-8"))) + + def test_get_activated_fetcher(self): + + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/fetchers") - self.assertIn("fetcher", json.loads(response.data.decode("utf-8"))) + response_as_json = json.loads(response.data.decode("utf-8")) + fetcher_name = response_as_json["fetchers"][0]["name"] + self.assertEqual("kiskadee-fetcher", fetcher_name) if __name__ == '__main__': From 0f528c9a845b0a4c90f1ef84e8da629bb244049f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:56 +0000 Subject: [PATCH 48/54] Try to uncompress the source file. --- diff --git a/kiskadee/runner.py b/kiskadee/runner.py index e115353..50757e4 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -131,11 +131,8 @@ class Runner: ) ) uncompressed_source_path = tempfile.mkdtemp() - shutil.unpack_archive(compressed_source, uncompressed_source_path) - kiskadee.logger.debug( - 'ANALYSIS: Unpacking {} source in {} path' - .format(package['name'], uncompressed_source_path) - ) + try: + shutil.unpack_archive(compressed_source, uncompressed_source_path) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' .format(package['name'], uncompressed_source_path) From 90965bf4faa5a5a356d93dd4b84aecb97f79566f Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:56 +0000 Subject: [PATCH 49/54] Update README.md --- diff --git a/README.md b/README.md index b030bc2..a572479 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,14 @@ into a Firehose database. ## Setup -### Dependencies +To run kiskadee, you must have docker installed and running. Use the +dockerfiles in the `util` directory to build the images for each static +analyzer. The name of the image must be equal of the analyzer name. +You can accomplish that by doing -The name of the dependencies are compatible -with the Fedora distribution. If you use another operational system, -you will have to find the compatible names for the dependencies. -The `redhat-rpm-config` -package, is a specific Fedora dependency, if you not use Fedora (or a -Red Hat distribution), maybe you will not have to install it. + docker build . -t cppcheck + +With the Docker images build, create a virtualenv to kiskadee `dnf` is a package manager for the Fedora distribution (On Debian and Ubuntu is apt), @@ -25,7 +25,11 @@ to install the dependencies below. - redhat-rpm-config python-pip - python-pip -### Virtual Environment +Install some package dependencies. The name of the dependencies are compatible +with the Fedora distribution. If you use another distribution, you will have +to find the compatible name for the dependencies. The `redhat-rpm-config` +package, is a specific Fedora dependency. If you are not in Fedora (or a +Red Hat distribution), maybe you will not have to install it. Create a [virtualenv](https://virtualenv.pypa.io/en/stable/) to kiskadee. The virtualenv package will create a isolated environment @@ -35,94 +39,14 @@ for our python dependencies. virtualenv -p /usr/bin/python3 . source bin/activate -Install the python dependencies using pip +Kiskadee use postgresql as database. You will need to create a database named +kiskadee, with a role kiskadee as owner. + +Install python dependencies and run kiskadee pip install -e . pip install "fedmsg[consumers]" -### Docker Images - -To run the static analyzers, you must have -[Docker](https://www.docker.com/community-edition) installed and running. -If you have configured the Docker engineer properly, -run the *docker_build.sh* script. It will build the images for you. - - chmod u+x docker_build.sh - ./docker_build.sh - -### Database -Now we will create the kiskadee database. You will need to install the -postgresql packages for your system. If you use Fedora, follow the next -steps, if not, you will have to find out how install postgresql on your -system. - - sudo dnf install postgresql-server postgresql-contrib - sudo systemctl enable postgresql - sudo postgresql-setup initdb - sudo systemctl start postgresql - -To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). - -With postgresql installed, you will need to create the kiskadee role and -database. - - sudo su - postgres - createdb kiskadee - createuser kiskadee -P - # use kiskadee as password. - psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" - # go back to your user (ctrl+d) - echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass - chmod 600 ~/.pgpass - -Restart the postgresql service: - - sudo systemctl restart postgresql - -Test the database connection: - - psql -U kiskadee -d kiskadee - -If you was not able to log in on the database, you will need to edit -the *pg_hba.conf* and change some rules defined by the postgresql package. -On Linux systems this file normally stays at the -`/var/lib/pgsql/data/`. Open this file and change: - - # "local" is for Unix domain socket connections only - local all all peer - # IPv4 local connections: - host all all 127.0.0.1/32 ident - # IPv6 local connections: - host all all ::1/128 ident - -to: - - # "local" is for Unix domain socket connections only - local all all md5 - # IPv4 local connections: - host all all 127.0.0.1/32 md5 - # IPv6 local connections: - host all all ::1/128 md5 - - -After this change, restarts the postgresql service: - - sudo systemctl restart postgresql - -Test the database connection: - - psql -U kiskadee -d kiskadee - -If you was able to get into the psql shell, the database is properly -configured. Leave the shell with ctrl+d. - -### Running our first analysis - -kiskadee reads environment variables from the `util/kiskadee.conf` file. -If everything goes well till now, open the *kiskadee.conf* file, and set as -active (`active = yes`) only the *example_fetcher*, the other fetchers will -stay as `active = no`. - Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a @@ -132,33 +56,50 @@ kiskadee will decompress the example source, and run the analyzers defined on the *kiskadee.conf* file. You can use any postgresql client to access the database that you have created, and check the analysis maded by kiskadee. -### Running API +Kiskadee looks for its configuration file under `util/kiskadee.conf`. +If everything goes well till now, open the kiskadee.conf file, and set as +active only the example fetcher. Now run kiskadee by typing `kiskadee` on +the terminal. If the Docker images was properly build, and the Docker client +was properly configured on your machine, kiskadee will be able to analysis a +exemple source code. This code is in the kiskadee/tests/test\_source/ directory. -To run the kiskadee api just execute the command: +To run the API just run the command `kiskadee_api`. - kiskadee_api +### Anitya Fetcher +If you intend to run the anitya fetcher, you will have to install fedmsg-hub, +in order to kiskadee be able to consume the fedmsg events. +To install fedmsg-hub follow this steps inside the kiskadee root path: -## Tests and coverage + # Run this inside the kiskadee's virtualenv + sudo mkdir -p /etc/fedmsg.d/ + sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ + sudo cp util/anityaconsumer.py /etc/fedmsg.d/ + PYTHONPATH=`pwd` fedmsg-hub -To check kiskadee tests and coverage just run: +With this steps, fedmsg-hub will instantiate `AnityaConsumer` and publish +the monitored events using ZeroMQ. When kiskadee starts it will consume +the messages published by the consumer, and will run the analysis. - chmod u+x run_tests_and_coverage.sh - ./run_tests_and_coverage.sh +The events that comes to the anitya fetcher are published by Anitya, on this +[page](https://apps.fedoraproject.org/datagrepper/raw?category=anitya.) -To check kiskadee coverage open the file *covhtml/index.html*. +For more info about the Anitya service, read kiskadee documentation. -## Repositories +### Debian Fetcher +If you intend to use the debian fetcher, you will have to install the +`devscripts` package, in order use the necessary Debian tools to run the +fetcher. + +## Development kiskadee daemon and API development are hosted at [pagure](https://pagure.io/kiskadee). kiskadee frontend is hosted at [pagure](https://pagure.io/kiskadee/kiskadee_ui). Feel free to open issues and pull requests there. -We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and +We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and [github](https://github.com/LSS-USP/kiskadee). -kiskadee have a CI environment hosted at this [url](http://143.107.45.126:30130/blue/organizations/jenkins/LSS-USP%2Fkiskadee/activity). - ## Documentation [kiskadee documentation is hosted at pagure.](docs.pagure.org/kiskadee) @@ -167,36 +108,9 @@ To build the documentation just entry in the doc directory, and run make html -To access the documentation open the `index.html` file, inside the +To access the documentation open the `index.html` file, inside the doc/\_build/html. -## Fetchers - -### Debian Fetcher -If you intend to use the debian fetcher, you will have to install the -`devscripts` package, in order use the necessary Debian tools to run the -fetcher. - -### Anitya Fetcher -If you intend to run the anitya fetcher, you will have to install fedmsg-hub, -in order to kiskadee be able to consume the fedmsg events. -To install fedmsg-hub follow this steps inside the kiskadee root path: - - # Run this inside the kiskadee's virtualenv - sudo mkdir -p /etc/fedmsg.d/ - sudo cp util/base.py util/endpoints.py /etc/fedmsg.d/ - sudo cp util/anityaconsumer.py /etc/fedmsg.d/ - PYTHONPATH=`pwd` fedmsg-hub - -With this steps, fedmsg-hub will instantiate `AnityaConsumer` and publish -the monitored events using ZeroMQ. When kiskadee starts it will consume -the messages published by the consumer, and will run the analysis. - -The events that comes to the anitya fetcher are published by Anitya, on this -[page](https://apps.fedoraproject.org/datagrepper/raw?category=anitya.) -For more info about the Anitya service, read kiskadee documentation. - - ## License Copyright (C) 2017 the AUTHORS (see the AUTHORS file) diff --git a/doc/architecture.rst b/doc/architecture.rst index 7af09aa..579cc48 100644 --- a/doc/architecture.rst +++ b/doc/architecture.rst @@ -26,4 +26,3 @@ kiskadee authors. .. *Figure One: Kiskadee architecture.* - diff --git a/kiskadee/runner.py b/kiskadee/runner.py index 50757e4..f3b6b80 100644 --- a/kiskadee/runner.py +++ b/kiskadee/runner.py @@ -132,7 +132,10 @@ class Runner: ) uncompressed_source_path = tempfile.mkdtemp() try: - shutil.unpack_archive(compressed_source, uncompressed_source_path) + shutil.unpack_archive( + compressed_source, + uncompressed_source_path + ) kiskadee.logger.debug( 'ANALYSIS: Unpacking {} source in {} path' .format(package['name'], uncompressed_source_path) From a966c3ab72c155a407f8bba5a09b455167755e9e Mon Sep 17 00:00:00 2001 From: David Carlos Date: Aug 29 2017 19:33:56 +0000 Subject: [PATCH 50/54] Add CI link in README.me --- diff --git a/README.md b/README.md index a572479..cd43dec 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ Feel free to open issues and pull requests there. We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and [github](https://github.com/LSS-USP/kiskadee). +Kiskadee have a CI environment hosted at this [url](http://143.107.45.126:30130/blue/organizations/jenkins/LSS-USP%2Fkiskadee/activity). ## Documentation [kiskadee documentation is hosted at pagure.](docs.pagure.org/kiskadee) From 46b7962117585a9fb12479c780fb89e98d0dc3a2 Mon Sep 17 00:00:00 2001 From: gabrielsclimaco Date: Aug 29 2017 19:33:56 +0000 Subject: [PATCH 51/54] Add docker for environment set up and documentation for running it --- diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9414382 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9cb82e8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM fedora + +RUN curl -o docker.rpm https://download.docker.com/linux/fedora/24/x86_64/stable/Packages/docker-ce-17.06.0.ce-1.fc24.x86_64.rpm &&\ + dnf install -y openssl-devel python3-devel gcc redhat-rpm-config python-pip docker.rpm &&\ + mkdir /app + +ADD . /app +WORKDIR /app + +RUN pip install virtualenv && virtualenv -p /usr/bin/python3 . &&\ + source bin/activate && pip install -e . && pip install "fedmsg[consumers]" + +RUN source bin/activate diff --git a/README.md b/README.md index cd43dec..8774b67 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,125 @@ Install python dependencies and run kiskadee pip install -e . pip install "fedmsg[consumers]" +### Docker Images + +To run the static analyzers, you must have +[Docker](https://www.docker.com/community-edition) installed and running. +If you have configured the Docker engineer properly, +run the *docker_build.sh* script. It will build the images for you. + + chmod u+x docker_build.sh + ./docker_build.sh + +### Database +Now we will create the kiskadee database. You will need to install the +postgresql packages for your system. If you use Fedora, follow the next +steps, if not, you will have to find out how install postgresql on your +system. + + sudo dnf install postgresql-server postgresql-contrib + sudo systemctl enable postgresql + sudo postgresql-setup initdb + sudo systemctl start postgresql + +To install on Ubuntu use this [link](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-postgresql-on-ubuntu-16-04). + +With postgresql installed, you will need to create the kiskadee role and +database. + + sudo su - postgres + createdb kiskadee + createuser kiskadee -P + # use kiskadee as password. + psql -U postgres -c "grant all privileges on database kiskadee to kiskadee" + # go back to your user (ctrl+d) + echo "localhost:5432:kiskadee:kiskadee:kiskadee" > ~/.pgpass + chmod 600 ~/.pgpass + +Restart the postgresql service: + + sudo systemctl restart postgresql + +Test the database connection: + + psql -U kiskadee -d kiskadee + +If you was not able to log in on the database, you will need to edit +the *pg_hba.conf* and change some rules defined by the postgresql package. +On Linux systems this file normally stays at the +`/var/lib/pgsql/data/`. Open this file and change: + + # "local" is for Unix domain socket connections only + local all all peer + # IPv4 local connections: + host all all 127.0.0.1/32 ident + # IPv6 local connections: + host all all ::1/128 ident + +to: + + # "local" is for Unix domain socket connections only + local all all md5 + # IPv4 local connections: + host all all 127.0.0.1/32 md5 + # IPv6 local connections: + host all all ::1/128 md5 + + +After this change, restarts the postgresql service: + + sudo systemctl restart postgresql + +Test the database connection: + + psql -U kiskadee -d kiskadee + +If you was able to get into the psql shell, the database is properly +configured. Leave the shell with ctrl+d. + +### With Docker + +If you don't want to install all dependencies, use the Dockerfile on the root of the project: + +1. First, build the image: + +``` +docker build -t kiskadee_backend . +``` + +2. Then, change the execution permissions of the docker shell script: + +``` +chmod +x run_docker.sh +``` + +* 3. Run the shell script + +``` +./run_docker.sh +``` + +* 4. Now you're into docker, just enter the environment as usual: + +``` +source bin/activate +``` + +Now you can run ```kiskadee```. + +--- + +**Obs:** You still need to set up the [PostgreSQL database](#database) and build the [docker images](#docker-images). + +--- + +### Running our first analysis + +Kiskadee reads environment variables from the `util/kiskadee.conf` file. +If everything goes well till now, open the *kiskadee.conf* file, and set as +active (`active = yes`) only the *example_fetcher*, the other fetchers will +stay as `active = no`. + Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client was properly configured on your machine, kiskadee will be able to analysis a diff --git a/run_docker.sh b/run_docker.sh new file mode 100755 index 0000000..cd82d24 --- /dev/null +++ b/run_docker.sh @@ -0,0 +1,13 @@ +docker run --rm -it \ + -v "/var/run/docker.sock:/var/run/docker.sock" \ + -v "$(pwd)/kiskadee:/app/kiskadee" \ + -v "$(pwd)/util:/app/util" \ + --net="host" \ + kiskadee_backend bash + +# line 1 - Run container iteratively without saving its instance +# line 2 - Map docker.sock to make docker in docker possible +# line 3 - Map kiskadee folder +# line 4 - Map util folder +# line 5 - Make sure the container run in the same host as the docker to connect +# with PostgreSQL via port 5432 From c017afda6030f4975cb54454fc426d05b040a0e2 Mon Sep 17 00:00:00 2001 From: gabrielsclimaco Date: Aug 29 2017 19:33:56 +0000 Subject: [PATCH 52/54] Remove shell script to run docker --- diff --git a/run_docker.sh b/run_docker.sh deleted file mode 100755 index cd82d24..0000000 --- a/run_docker.sh +++ /dev/null @@ -1,13 +0,0 @@ -docker run --rm -it \ - -v "/var/run/docker.sock:/var/run/docker.sock" \ - -v "$(pwd)/kiskadee:/app/kiskadee" \ - -v "$(pwd)/util:/app/util" \ - --net="host" \ - kiskadee_backend bash - -# line 1 - Run container iteratively without saving its instance -# line 2 - Map docker.sock to make docker in docker possible -# line 3 - Map kiskadee folder -# line 4 - Map util folder -# line 5 - Make sure the container run in the same host as the docker to connect -# with PostgreSQL via port 5432 From 6eabf1f71173dc99312131105f5cf1cbff7e129a Mon Sep 17 00:00:00 2001 From: gabrielsclimaco Date: Aug 29 2017 19:44:45 +0000 Subject: [PATCH 53/54] Update README due to removing docker environment shell script --- diff --git a/README.md b/README.md index 8774b67..aa11250 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ to install the dependencies below. Install some package dependencies. The name of the dependencies are compatible with the Fedora distribution. If you use another distribution, you will have -to find the compatible name for the dependencies. The `redhat-rpm-config` +to find the compatible name for the dependencies. The `redhat-rpm-config` package, is a specific Fedora dependency. If you are not in Fedora (or a Red Hat distribution), maybe you will not have to install it. @@ -127,25 +127,24 @@ configured. Leave the shell with ctrl+d. If you don't want to install all dependencies, use the Dockerfile on the root of the project: -1. First, build the image: +#### First, build the image: ``` docker build -t kiskadee_backend . ``` -2. Then, change the execution permissions of the docker shell script: +#### Then, run the image using the following command: ``` -chmod +x run_docker.sh +docker run —rm -it \ + -v "/var/run/docker.sock:/var/run/docker.sock" \ + -v "$(pwd)/kiskadee:/app/kiskadee" \ + -v "$(pwd)/util:/app/util" \ + —net="host" \ + kiskadee_backend bash ``` -* 3. Run the shell script - -``` -./run_docker.sh -``` - -* 4. Now you're into docker, just enter the environment as usual: +#### Now you're into docker, just enter the environment as usual: ``` source bin/activate @@ -175,7 +174,7 @@ kiskadee will decompress the example source, and run the analyzers defined on the *kiskadee.conf* file. You can use any postgresql client to access the database that you have created, and check the analysis maded by kiskadee. -Kiskadee looks for its configuration file under `util/kiskadee.conf`. +Kiskadee looks for its configuration file under `util/kiskadee.conf`. If everything goes well till now, open the kiskadee.conf file, and set as active only the example fetcher. Now run kiskadee by typing `kiskadee` on the terminal. If the Docker images was properly build, and the Docker client @@ -216,7 +215,7 @@ kiskadee daemon and API development are hosted at [pagure](https://pagure.io/kis kiskadee frontend is hosted at [pagure](https://pagure.io/kiskadee/kiskadee_ui). Feel free to open issues and pull requests there. -We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and +We also have mirrors on [gitlab](https://gitlab.com/kiskadee/kiskadee) and [github](https://github.com/LSS-USP/kiskadee). Kiskadee have a CI environment hosted at this [url](http://143.107.45.126:30130/blue/organizations/jenkins/LSS-USP%2Fkiskadee/activity). @@ -228,7 +227,7 @@ To build the documentation just entry in the doc directory, and run make html -To access the documentation open the `index.html` file, inside the +To access the documentation open the `index.html` file, inside the doc/\_build/html. ## License From 0414210a7b644e317f9d80de2be1033103a6bc84 Mon Sep 17 00:00:00 2001 From: gabrielsclimaco Date: Aug 29 2017 19:55:45 +0000 Subject: [PATCH 54/54] Fix typo on readme --- diff --git a/README.md b/README.md index aa11250..881d0d4 100644 --- a/README.md +++ b/README.md @@ -136,11 +136,11 @@ docker build -t kiskadee_backend . #### Then, run the image using the following command: ``` -docker run —rm -it \ +docker run --rm -it \ -v "/var/run/docker.sock:/var/run/docker.sock" \ -v "$(pwd)/kiskadee:/app/kiskadee" \ -v "$(pwd)/util:/app/util" \ - —net="host" \ + --net="host" \ kiskadee_backend bash ```