From 4814f9dde7b96155ba10864893ba80936d7b075a Mon Sep 17 00:00:00 2001 From: Vitor Borges Date: Dec 07 2017 12:41:50 +0000 Subject: [PATCH 1/3] [API] Refactoring api endpoints for analysis Signed-off-by: MatheusGodinho Signed-off-by: Vitor Borges --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 4acdb24..b80ee10 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -36,33 +36,66 @@ def packages(): return jsonify({'packages': result.data}) -@kiskadee.route('/analysis//') -def package_analysis(pkg_name, version): - """Get the a analysis of some package version.""" - if request.method == 'GET': - db_session = kiskadee_db_session() - package_id = ( - db_session.query(Package) - .filter(Package.name == pkg_name).first().id - ) - version_id = ( - db_session.query(Version) - .filter(Version.number == version) - .filter(Version.package_id == package_id).first().id - ) - analysis = ( - db_session.query(Analysis) - .filter(Analysis.version_id == version_id).all() - ) - analysis_schema = AnalysisSchema(many=True) - results = analysis_schema.dump(analysis) - for result in results.data: - report = result['report'] - if (report is not None) and ('results' in report.keys()): - report['results'] = json\ - .loads(report['results']) - return jsonify({'analysis': results.data}) +@kiskadee.route('/analysis//', methods=['GET']) +def package_analysis_overview(pkg_name, version): + """Get the a analysis list of some package version.""" + db_session = kiskadee_db_session() + package_id = ( + db_session.query(Package) + .filter(Package.name == pkg_name).first().id + ) + version_id = ( + db_session.query(Version) + .filter(Version.number == version) + .filter(Version.package_id == package_id).first().id + ) + analysis = ( + db_session.query(Analysis) + .filter(Analysis.version_id == version_id).all() + ) + analysis_schema = AnalysisSchema(many=True) + results = analysis_schema.dump(analysis) + result_data = [] + for result in results.data: + current_data = { + 'analyzer_id': result['analyzer_id'], + 'id': result['id'], + 'version': result['raw']['metadata']['generator']['version'], + 'name': result['raw']['metadata']['generator']['name'] + } + result_data.append(current_data) + return jsonify({'analysis': result_data}) + + +@kiskadee.route('/analysis////results', methods=['GET']) +def analysis_results(pkg_name, version, analysis_id): + """Get the analysis results from a specific analyzer.""" + db_session = kiskadee_db_session() + analysis = ( + db_session.query(Analysis) + .get(analysis_id) + ) + analysis_schema = AnalysisSchema() + results = analysis_schema.dump(analysis) + response = results.data['raw']['results'] + return jsonify({'analysis_results': response}) + +@kiskadee.route('/analysis////reports', methods=['GET']) +def analysis_reports(pkg_name, version, analysis_id): + """Get the analysis results from a specific analyzer.""" + db_session = kiskadee_db_session() + analysis = ( + db_session.query(Analysis) + .get(analysis_id) + ) + analysis_schema = AnalysisSchema() + results = analysis_schema.dump(analysis) + report = results.data['report'] + if (report is not None) and ('results' in report.keys()): + report['results'] = json\ + .loads(report['results']) + return jsonify({'analysis_report': report}) def kiskadee_db_session(): """Return a kiskadee database session.""" From ec0f28ad2cc5e4a437a5b648630007948dae02b3 Mon Sep 17 00:00:00 2001 From: Vitor Borges Date: Dec 07 2017 12:41:50 +0000 Subject: [PATCH 2/3] Change `/package` router result Signed-off-by: DaniloBarros Signed-off-by: thiagocmoreira Signed-off-by: MatheusGodinho Signed-off-by: Vitor Borges --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index b80ee10..57e716b 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -31,9 +31,20 @@ def packages(): if request.method == 'GET': db_session = kiskadee_db_session() packages = db_session.query(Package).all() - package_schema = PackageSchema(many=True) - result = package_schema.dump(packages) - return jsonify({'packages': result.data}) + package_schema = PackageSchema( + many=True, + exclude=['versions.analysis', 'versions.package_id'] + ) + data, errors = package_schema.dump(packages) + # TODO: get the last version using sequelize + for item in data: + item['version'] = sorted( + item['versions'], + key=lambda k: k['number'], + reverse=True + )[0]['number'] # only the last version + item.pop('versions', None) + return jsonify({'packages': data}) @kiskadee.route('/analysis//', methods=['GET']) @@ -64,10 +75,13 @@ def package_analysis_overview(pkg_name, version): 'name': result['raw']['metadata']['generator']['name'] } result_data.append(current_data) - return jsonify({'analysis': result_data}) + return jsonify(result_data) -@kiskadee.route('/analysis////results', methods=['GET']) +@kiskadee.route( + '/analysis////results', + methods=['GET'] +) def analysis_results(pkg_name, version, analysis_id): """Get the analysis results from a specific analyzer.""" db_session = kiskadee_db_session() @@ -81,9 +95,12 @@ def analysis_results(pkg_name, version, analysis_id): return jsonify({'analysis_results': response}) -@kiskadee.route('/analysis////reports', methods=['GET']) +@kiskadee.route( + '/analysis////reports', + methods=['GET'] +) def analysis_reports(pkg_name, version, analysis_id): - """Get the analysis results from a specific analyzer.""" + """Get the analysis reports from a specific analyzer.""" db_session = kiskadee_db_session() analysis = ( db_session.query(Analysis) @@ -92,11 +109,14 @@ def analysis_reports(pkg_name, version, analysis_id): analysis_schema = AnalysisSchema() results = analysis_schema.dump(analysis) report = results.data['report'] - if (report is not None) and ('results' in report.keys()): + if (report is not None) and\ + ('results' in report.keys()) and\ + report['results'] is not None: report['results'] = json\ .loads(report['results']) return jsonify({'analysis_report': report}) + def kiskadee_db_session(): """Return a kiskadee database session.""" return Database().session diff --git a/kiskadee/tests/test_api.py b/kiskadee/tests/test_api.py index d3714c1..71a58fb 100644 --- a/kiskadee/tests/test_api.py +++ b/kiskadee/tests/test_api.py @@ -92,12 +92,26 @@ class ApiTestCase(unittest.TestCase): kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session response = self.app.get("/analysis/test/1.0.0") response_data = json.loads(response.data.decode("utf-8")) - pkg_first_analysis = response_data["analysis"][0] - self.assertIsNotNone(pkg_first_analysis["raw"]) - self.assertIn('report', pkg_first_analysis) - self.assertIn('location', pkg_first_analysis["raw"]["results"][0]) - self.assertIn('cwe', pkg_first_analysis["raw"]["results"][0]) - self.assertIn('message', pkg_first_analysis["raw"]["results"][0]) + self.assertTrue(len(response_data) >= 1) + + def test_get_analysis_results(self): + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session + response = self.app.get("/analysis/kiskadee-package/7.23/1/results") + response_data = json.loads(response.data.decode("utf-8")) + self.assertIn("analysis_results", response_data) + self.assertTrue(len(response_data["analysis_results"]) > 0) + + def test_get_analysis_reports(self): + def mock_kiskadee_db_session(): + return self.session + + kiskadee.api.app.kiskadee_db_session = mock_kiskadee_db_session + response = self.app.get("/analysis/kiskadee-package/7.23/1/reports") + response_data = json.loads(response.data.decode("utf-8")) + self.assertIn("analysis_report", response_data) if __name__ == '__main__': From a565685f603b36cb91574bdb408af4b88a509636 Mon Sep 17 00:00:00 2001 From: Vitor Borges Date: Dec 07 2017 12:48:26 +0000 Subject: [PATCH 3/3] Add AnalyzerSchema to serializers --- diff --git a/kiskadee/api/app.py b/kiskadee/api/app.py index 57e716b..2564bf2 100644 --- a/kiskadee/api/app.py +++ b/kiskadee/api/app.py @@ -8,6 +8,7 @@ from kiskadee.model import Package, Fetcher, Version, Analysis from kiskadee.api.serializers import PackageSchema, FetcherSchema,\ AnalysisSchema import json +from sqlalchemy.orm import eagerload kiskadee = Flask(__name__) @@ -36,14 +37,6 @@ def packages(): exclude=['versions.analysis', 'versions.package_id'] ) data, errors = package_schema.dump(packages) - # TODO: get the last version using sequelize - for item in data: - item['version'] = sorted( - item['versions'], - key=lambda k: k['number'], - reverse=True - )[0]['number'] # only the last version - item.pop('versions', None) return jsonify({'packages': data}) @@ -62,20 +55,15 @@ def package_analysis_overview(pkg_name, version): ) analysis = ( db_session.query(Analysis) - .filter(Analysis.version_id == version_id).all() + .options( + eagerload(Analysis.analyzers, innerjoin=True) + ) + .filter(Analysis.version_id == version_id) + .all() ) - analysis_schema = AnalysisSchema(many=True) - results = analysis_schema.dump(analysis) - result_data = [] - for result in results.data: - current_data = { - 'analyzer_id': result['analyzer_id'], - 'id': result['id'], - 'version': result['raw']['metadata']['generator']['version'], - 'name': result['raw']['metadata']['generator']['name'] - } - result_data.append(current_data) - return jsonify(result_data) + analysis_schema = AnalysisSchema(many=True, exclude=['raw', 'report']) + data, errors = analysis_schema.dump(analysis) + return jsonify(data) @kiskadee.route( @@ -89,9 +77,9 @@ def analysis_results(pkg_name, version, analysis_id): db_session.query(Analysis) .get(analysis_id) ) - analysis_schema = AnalysisSchema() - results = analysis_schema.dump(analysis) - response = results.data['raw']['results'] + analysis_schema = AnalysisSchema(only=['raw']) + data, errors = analysis_schema.dump(analysis) + response = data['raw']['results'] return jsonify({'analysis_results': response}) @@ -106,9 +94,9 @@ def analysis_reports(pkg_name, version, analysis_id): db_session.query(Analysis) .get(analysis_id) ) - analysis_schema = AnalysisSchema() - results = analysis_schema.dump(analysis) - report = results.data['report'] + analysis_schema = AnalysisSchema(only=['report']) + data, errors = analysis_schema.dump(analysis) + report = data['report'] if (report is not None) and\ ('results' in report.keys()) and\ report['results'] is not None: diff --git a/kiskadee/api/serializers.py b/kiskadee/api/serializers.py index 98e7835..e7168e9 100644 --- a/kiskadee/api/serializers.py +++ b/kiskadee/api/serializers.py @@ -1,7 +1,8 @@ """Provide objects to serialize the kiskadee models.""" from marshmallow import Schema, fields -from kiskadee.model import Package, Fetcher, Analysis, Version, Report +from kiskadee.model import Package, Fetcher, Analysis, Version,\ + Report, Analyzer class ReportsSchema(Schema): @@ -17,6 +18,20 @@ class ReportsSchema(Schema): return Report(**data) +class AnalyzerSchema(Schema): + """Provide a serializer to the Analyzer model.""" + + id = fields.Int() + name = fields.Str() + version = fields.Str() + analysis = fields.Nested('AnalysisSchema', many=True) + + def make_object(self, data): + """Serialize a Analyzer object.""" + print('MAKING OBJECT FRON', data) + return Analyzer(**data) + + class AnalysisSchema(Schema): """Provide a serializer to the Analysis model.""" @@ -25,6 +40,7 @@ class AnalysisSchema(Schema): analyzer_id = fields.Int() raw = fields.Dict() report = fields.Nested(ReportsSchema) + analyzers = fields.Nested(AnalyzerSchema, exclude=['analysis', 'id']) def make_object(self, data): """Serialize a Analysis object."""