diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26563283c..f9b39c0a3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Release notes ============= +next release +--------------------- + +- WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. + Version v38.6.0 --------------------- diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index c0cf04ed7..dd2c504ce 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -28,7 +28,6 @@ from vulnerabilities.importers import retiredotnet from vulnerabilities.importers import ruby from vulnerabilities.importers import suse_scores -from vulnerabilities.importers import ubuntu from vulnerabilities.importers import ubuntu_usn from vulnerabilities.importers import vulnrichment from vulnerabilities.importers import xen @@ -137,7 +136,6 @@ openssl_importer_v2.OpenSSLImporterPipeline, redhat.RedhatImporter, archlinux.ArchlinuxImporter, - ubuntu.UbuntuImporter, debian_oval.DebianOvalImporter, retiredotnet.RetireDotnetImporter, apache_httpd.ApacheHTTPDImporter, diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py deleted file mode 100644 index e47515b93..000000000 --- a/vulnerabilities/importers/ubuntu.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import bz2 -import logging -import xml.etree.ElementTree as ET - -import requests - -from vulnerabilities.importer import OvalImporter - -logger = logging.getLogger(__name__) - - -class UbuntuImporter(OvalImporter): - spdx_license_expression = "LicenseRef-scancode-other-permissive" - notice = """ - From: Seth Arnold - Date: Wed, Jan 25, 2023 at 2:02 AM - Subject: Re: [ubuntu-hardened] Usage of Ubuntu Security Data in VulnerableCode - To: Tushar Goel - Cc: , Philippe Ombredanne , jmhoran@nexb.com - - - On Wed, Jan 11, 2023 at 06:27:38PM +0530, Tushar Goel wrote: - > We would like to integrate the Ubuntu usn data[1][2] and - > Ubuntu security data (OVAL format)[3] in vulnerablecode[4] - > which is a FOSS db of FOSS vulnerability data. We were not - > able to know under which license this security data comes. - > We would be grateful to have your acknowledgement over usage of - > the ubuntu security data in vulnerablecode and have - > some kind of licensing declaration from your side. - - Hello Tushar, we do not have an explicit license on this data. - - We share our data with the intention that others will use it. Please - feel free to use it for the general furtherance of security. - - Much of the data that's contained within our databases is sourced from - third parties, who also shared their data with the intention that others - will use it. I'm not sure what it would look like to try to put a license - on data that is crowd-sourced from thousands of contributors. (If you were - to start such a project today, it'd probably be one of the first things to - formalize. But when CVE was started two decades ago, the primary goal was - sharing knowledge and simplifying the vulnerability remediation process, - and licensing the data was, as far as I can remember, not considered. - Sharing was the goal.) - - I will ask that vulnerablecode 'be nice' to our infrastructure that - hosts the databases -- some automated uses of our infrastructure by - vulnerability scanner tools has lead to significant load and engineering - effort. In general, please prefer a small handful of systems updating - mirrors roughly twice a day rather than thousands of hosts pulling - data hourly. - - Thanks - """ - importer_name = "Ubuntu OVAL Importer" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # we could avoid setting translations, and have it - # set by default in the OvalParser, but we don't yet know - # whether all OVAL providers use the same format - self.translations = {"less than": "<"} - - def _fetch(self): - base_url = "https://people.canonical.com/~ubuntu-security/oval" - releases = ["bionic", "trusty", "focal", "eoan", "xenial"] - for release in releases: - file_url = f"{base_url}/com.ubuntu.{release}.cve.oval.xml.bz2" # nopep8 - self.data_url = file_url - logger.info(f"Fetching Ubuntu Oval: {file_url}") - response = requests.get(file_url) - if response.status_code != requests.codes.ok: - logger.error( - f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}" - ) - continue - - extracted = bz2.decompress(response.content) - yield ( - {"type": "deb", "namespace": "ubuntu"}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index c55c14c8a..0dcd10563 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -48,7 +48,6 @@ valid_versions.ApacheKafkaImprover, valid_versions.IstioImprover, valid_versions.DebianOvalImprover, - valid_versions.UbuntuOvalImprover, valid_versions.OSSFuzzImprover, valid_versions.RubyImprover, valid_versions.GithubOSVImprover, diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 916f36f59..6fe102fae 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -35,7 +35,6 @@ from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter from vulnerabilities.importers.ruby import RubyImporter -from vulnerabilities.importers.ubuntu import UbuntuImporter from vulnerabilities.improver import MAX_CONFIDENCE from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference @@ -458,11 +457,6 @@ class DebianOvalImprover(ValidVersionImprover): ignorable_versions = [] -class UbuntuOvalImprover(ValidVersionImprover): - importer = UbuntuImporter - ignorable_versions = [] - - class OSSFuzzImprover(ValidVersionImprover): importer = OSSFuzzImporter ignorable_versions = [] diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 51728b631..5d0fee2de 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -273,6 +273,7 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline): pipeline_id = None # Unique Pipeline ID, this should be the name of pipeline module. license_url = None + datasource_name = None spdx_license_expression = None repo_url = None ignorable_versions = [] diff --git a/vulnerabilities/pipelines/remove_duplicate_advisories.py b/vulnerabilities/pipelines/remove_duplicate_advisories.py index bb4d749b2..1a55507a6 100644 --- a/vulnerabilities/pipelines/remove_duplicate_advisories.py +++ b/vulnerabilities/pipelines/remove_duplicate_advisories.py @@ -18,6 +18,7 @@ class RemoveDuplicateAdvisoriesPipeline(VulnerableCodePipeline): """Pipeline to compute new advisory content id and remove duplicate advisories based on their content.""" pipeline_id = "remove_duplicate_advisories" + run_once = True @classmethod def steps(cls): diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py deleted file mode 100644 index b31588347..000000000 --- a/vulnerabilities/tests/test_ubuntu.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -import xml.etree.ElementTree as ET -from unittest.mock import patch - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.ubuntu import UbuntuImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import UbuntuOvalImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") - - -def test_ubuntu_importer(): - importer = UbuntuImporter() - advsiories = importer.get_data_from_xml_doc( - ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")), - {"type": "deb", "namespace": "ubuntu"}, - ) - expected_file = os.path.join(TEST_DATA, f"ubuntu-oval-expected.json") - util_tests.check_results_against_json( - [advisory.to_dict() for advisory in advsiories], expected_file - ) - - -@patch("vulnerabilities.improvers.valid_versions.UbuntuOvalImprover.get_package_versions") -def test_debian_oval_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"ubuntu-oval-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [UbuntuOvalImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"ubuntu-oval-improver-expected.json") - util_tests.check_results_against_json(result, expected_file)