From 83a14c5081f43a0498e60ec22e9f088da421a870 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 15:48:23 +0200
Subject: [PATCH 01/14] Review code
---
files/gvm_library.py | 23 +++++++++++++----------
1 file changed, 13 insertions(+), 10 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index e78a9d4..ed8dbf6 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -11,6 +11,7 @@ import yaml
from functools import reduce
import os
import git
+import pandas as pd
# GVM Xpath Constants
GVM_XPATH_ID = '@id'
@@ -461,16 +462,18 @@ class Task:
with open(filename, "wb") as fh:
fh.write(base64.b64decode(code))
- def get_report(self) -> List[Tuple[str,str,str,str]]:
+ def get_report(self) -> pd.DataFrame:
res = self.client.get_report(self.report_id,
report_format_id=ReportFormats.anonymous_xml,
ignore_pagination=True,
details="1")
- o_ids: list[str] = res.xpath('report/report/results/result/nvt/@oid')
- severities: list[str] = res.xpath('report/report/results/result/nvt/severities/@score')
+ oids: list[str] = res.xpath('report/report/results/result/nvt/@oid')
+ sev: list[str] = res.xpath('report/report/results/result/nvt/severities/@score')
treats: list[str] = res.xpath('report/report/results/result/threat/text()')
ports: list[str] = res.xpath('report/report/results/result/port/text()')
- return [ResultReport(o,s,t,p) for o,s,t,p in zip(o_ids, severities, treats, ports)]
+ df = pd.DataFrame({"oids": oids, "sev": sev, "threats": treats, "ports": ports})
+ df['sev'] = df['sev'].astype(float)
+ return df
class GVMClient():
"""
@@ -605,8 +608,8 @@ class ReportManager():
else:
self.os_oids = os_oids
try:
- self.os_all_oids = list(reduce(lambda x,y: x + y,
- os_oids.values()))
+ self.os_all_oids = tuple(set(reduce(lambda x,y: x + y,
+ os_oids.values())))
logging.debug("Imported os security oids")
logging.debug(pretty_json(self.os_all_oids))
@@ -614,7 +617,7 @@ class ReportManager():
logging.warning("Impossible extract oids from imported yaml")
self.os_all_oids = []
- def extract_oids(self, lines: List[str]) -> Set[str]:
+ def extract_oids(self, lines: List[str]) -> Tuple[str]:
oids: List[str] = list()
for line in lines:
line = line.strip()
@@ -622,7 +625,7 @@ class ReportManager():
if len(v_line := line.split(" ")[0]) > 0:
oids.append(v_line)
- return set(oids)
+ return tuple(set(oids))
def import_security_oids(self) -> None:
user = os.environ.get(self.SS_SEC_USER)
@@ -656,7 +659,7 @@ class ReportManager():
logging.debug("known oids")
logging.debug(pretty_json(self.known_oids))
- def import_report(self, host: str, report: List[ResultReport]):
+ def import_report(self, host: str, report: pd.DataFrame) -> None:
self.imported_oids[host] = report
def init_glob_vars(self):
@@ -695,7 +698,7 @@ class ReportManager():
self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: r.severity,
self.REPORT_THREAT: r.threat}
- def classify_reports(self) -> None:
+ def classify_reports(self) -> None: ######################################<<<<<<<<<<<<<<<<<<########
# Init global aggregated variables
self.init_glob_vars()
--
GitLab
From 47272ba737ab3ecdcac557abeaab70973bef87d1 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 16:33:32 +0200
Subject: [PATCH 02/14] Implement advanced vulnerability classification
---
files/scan.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/files/scan.py b/files/scan.py
index e2f98f2..e856838 100644
--- a/files/scan.py
+++ b/files/scan.py
@@ -168,9 +168,9 @@ for host,ports in endpoints.items():
"threat": f"Scan Error. task.id: {task.id}"}
report_manager.import_report(host,task_report)
- task.delete()
- target.delete()
- port_list.delete()
+ #task.delete()
+ #target.delete()
+ #port_list.delete()
report_manager.classify_reports()
logging.info("Reports")
--
GitLab
From 105bf8b95aa571179a6c91b856d309edcaee215a Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 17:41:56 +0200
Subject: [PATCH 03/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 23 +++++++++++------------
files/scan.py | 4 +---
2 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index ed8dbf6..011b29f 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -462,18 +462,17 @@ class Task:
with open(filename, "wb") as fh:
fh.write(base64.b64decode(code))
- def get_report(self) -> pd.DataFrame:
+ def get_report(self) -> Dict[str,Tuple]:
res = self.client.get_report(self.report_id,
report_format_id=ReportFormats.anonymous_xml,
ignore_pagination=True,
details="1")
- oids: list[str] = res.xpath('report/report/results/result/nvt/@oid')
- sev: list[str] = res.xpath('report/report/results/result/nvt/severities/@score')
- treats: list[str] = res.xpath('report/report/results/result/threat/text()')
- ports: list[str] = res.xpath('report/report/results/result/port/text()')
- df = pd.DataFrame({"oids": oids, "sev": sev, "threats": treats, "ports": ports})
- df['sev'] = df['sev'].astype(float)
- return df
+ oids: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/@oid'))
+ sev: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/severities/@score'))
+ threat: tuple[str] = tuple(res.xpath('report/report/results/result/threat/text()'))
+ ports: tuple[str] = tuple(res.xpath('report/report/results/result/port/text()'))
+ sev = tuple(map(float,sev))
+ return {"oids":oids, "severity":sev, "threat":threat, "ports":ports}
class GVMClient():
"""
@@ -652,16 +651,16 @@ class ReportManager():
with open(filename, 'r') as file:
known_oids += self.extract_oids(file.readlines())
- self.accepted_oids = accepted_oids
- self.known_oids = known_oids
+ self.accepted_oids = tuple(sorted(accepted_oids))
+ self.known_oids = tuple(sorted(known_oids))
logging.debug("accepted oids")
logging.debug(pretty_json(self.accepted_oids))
logging.debug("known oids")
logging.debug(pretty_json(self.known_oids))
- def import_report(self, host: str, report: pd.DataFrame) -> None:
+ def import_report(self, host: str, report: Dict[str,Tuple]) -> None:
self.imported_oids[host] = report
-
+
def init_glob_vars(self):
self.report = dict()
self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY,
diff --git a/files/scan.py b/files/scan.py
index e856838..37e66b9 100644
--- a/files/scan.py
+++ b/files/scan.py
@@ -164,9 +164,7 @@ for host,ports in endpoints.items():
filename = f"{report_filename}.txt")
report_manager.import_report(host,task.get_report())
else:
- task_report = {"severity": ReportManager.MAX_SEVERITY,
- "threat": f"Scan Error. task.id: {task.id}"}
- report_manager.import_report(host,task_report)
+ logging.warning(f"Scan Error. task.id: {task.id}")
#task.delete()
#target.delete()
--
GitLab
From b41f8c309ac2dd2568a79839418b0d25d389b17e Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 17:46:00 +0200
Subject: [PATCH 04/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 5 +++++
files/scan.py | 2 ++
2 files changed, 7 insertions(+)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 011b29f..2b7be5f 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -661,6 +661,11 @@ class ReportManager():
def import_report(self, host: str, report: Dict[str,Tuple]) -> None:
self.imported_oids[host] = report
+
+ def show_imported_reports(self) -> None:
+ logging.debug(pretty_json(self.imported_oids))
+
+
def init_glob_vars(self):
self.report = dict()
self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY,
diff --git a/files/scan.py b/files/scan.py
index 37e66b9..1d9a6f6 100644
--- a/files/scan.py
+++ b/files/scan.py
@@ -170,6 +170,8 @@ for host,ports in endpoints.items():
#target.delete()
#port_list.delete()
+report_manager.show_imported_reports()
+
report_manager.classify_reports()
logging.info("Reports")
logging.info(report_manager.get_summary())
--
GitLab
From 0145af993261ed50ae7c9945a9aa0b446b260e33 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 17:47:59 +0200
Subject: [PATCH 05/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 2b7be5f..9c05744 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -467,11 +467,11 @@ class Task:
report_format_id=ReportFormats.anonymous_xml,
ignore_pagination=True,
details="1")
- oids: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/@oid'))
+ oids: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/nvt/@oid')))
sev: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/severities/@score'))
- threat: tuple[str] = tuple(res.xpath('report/report/results/result/threat/text()'))
- ports: tuple[str] = tuple(res.xpath('report/report/results/result/port/text()'))
- sev = tuple(map(float,sev))
+ threat: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/threat/text()')))
+ ports: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/port/text()')))
+ sev = tuple(sorted(map(float,sev)))
return {"oids":oids, "severity":sev, "threat":threat, "ports":ports}
class GVMClient():
@@ -661,11 +661,9 @@ class ReportManager():
def import_report(self, host: str, report: Dict[str,Tuple]) -> None:
self.imported_oids[host] = report
-
def show_imported_reports(self) -> None:
logging.debug(pretty_json(self.imported_oids))
-
def init_glob_vars(self):
self.report = dict()
self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY,
--
GitLab
From c8411cb17f9fd320e9634f357f2a9457302090fc Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 17:48:30 +0200
Subject: [PATCH 06/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 9c05744..cb574e1 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -467,11 +467,11 @@ class Task:
report_format_id=ReportFormats.anonymous_xml,
ignore_pagination=True,
details="1")
- oids: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/nvt/@oid')))
+ oids: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/@oid'))
sev: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/severities/@score'))
- threat: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/threat/text()')))
- ports: tuple[str] = tuple(sorted(res.xpath('report/report/results/result/port/text()')))
- sev = tuple(sorted(map(float,sev)))
+ threat: tuple[str] = tuple(res.xpath('report/report/results/result/threat/text()'))
+ ports: tuple[str] = tuple(res.xpath('report/report/results/result/port/text()'))
+ sev = tuple(map(float,sev))
return {"oids":oids, "severity":sev, "threat":threat, "ports":ports}
class GVMClient():
--
GitLab
From a8063368f2fab6b51bcb65b6b3c156e4f4a3f701 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Tue, 28 May 2024 17:52:40 +0200
Subject: [PATCH 07/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index cb574e1..b216c6a 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -662,7 +662,7 @@ class ReportManager():
self.imported_oids[host] = report
def show_imported_reports(self) -> None:
- logging.debug(pretty_json(self.imported_oids))
+ logging.debug( json.dumps(self.imported_oids))
def init_glob_vars(self):
self.report = dict()
--
GitLab
From 21cb3c67795d53cec7f4947c391f03fe673b5ca8 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:14:01 +0200
Subject: [PATCH 08/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 145 +++++++++++++++++++++----------------------
files/scan.py | 2 -
2 files changed, 72 insertions(+), 75 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index b216c6a..1d25fb3 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -471,7 +471,7 @@ class Task:
sev: tuple[str] = tuple(res.xpath('report/report/results/result/nvt/severities/@score'))
threat: tuple[str] = tuple(res.xpath('report/report/results/result/threat/text()'))
ports: tuple[str] = tuple(res.xpath('report/report/results/result/port/text()'))
- sev = tuple(map(float,sev))
+ sev: tuple[float] = tuple(map(float,sev))
return {"oids":oids, "severity":sev, "threat":threat, "ports":ports}
class GVMClient():
@@ -538,6 +538,7 @@ class ReportManager():
REPORT_GLOBAL = "global"
REPORT_SEVERITY = "severity"
REPORT_THREAT = "threat"
+ REPORT_PORTS = "ports"
# OIDS Classes
OID_ACCEPTED = 'accepted-oids'
@@ -563,6 +564,13 @@ class ReportManager():
SS_SEC_ACCEPTED_FILES = ['accepted.txt']
SS_SEC_KNOWN_FILES = ['held.txt', 'new.txt', 'overridden.txt']
+ # Classification configuration
+ LABEL_COLUMN = "label"
+ LABEL_NEW_VULNS = "NEW"
+ LABEL_ACKNOWLEDGED_VULNS = "ACKNOWLEDGED"
+ LABEL_REJECTED_VULNS = "REJECTED"
+ LABEL_OS_RELATED_VULNS = "OS_RELATED"
+
def __init__(self, os_name: str, is_os: bool) -> None:
logging.info("Report Manager Iniziatation started...")
self.os_name = os_name
@@ -662,83 +670,79 @@ class ReportManager():
self.imported_oids[host] = report
def show_imported_reports(self) -> None:
- logging.debug( json.dumps(self.imported_oids))
-
- def init_glob_vars(self):
- self.report = dict()
- self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY,
- self.REPORT_THREAT: self.DEFAULT_THREAT}
- self.oids = dict()
-
- def init_host_vars(self,host: str):
- self.oids[host] = {self.OID_ACCEPTED: [],
- self.OID_DROPPED: [],
- self.OID_NEW: []}
+ logging.debug(json.dumps(self.imported_oids))
- if not self.is_os:
- self.oids[host][self.OID_OS] = []
- self.report[host] = dict()
- self.report[host][self.REPORT_GLOBAL] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY,
- self.REPORT_THREAT: self.DEFAULT_THREAT}
+ def classify_reports(self) -> None:
+ TO_SOLVE_VULNS = [self.LABEL_NEW_VULNS,self.LABEL_ACKNOWLEDGED_VULNS]
- def update_summary(self,host, r: ResultReport) -> None:
+ to_solve = pd.Series({"oids": self.accepted_oids})
+ to_exclude = pd.Series({"oids": self.known_oids})
+ os_vulns = pd.Series({"oids": self.os_all_oids})
- # Evaluate max port severity per host
- if r.port not in self.report[host] or \
- r.severity > self.report[host][r.port][self.REPORT_SEVERITY]:
- self.report[host][r.port] = {self.REPORT_SEVERITY: r.severity,
- self.REPORT_THREAT: r.threat}
+ self.report, self.oids = dict(), dict()
+ self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: self.DEFAULT_SEVERITY}
- # Evaluate max global severity per host
- if r.severity > self.report[host][self.REPORT_GLOBAL][self.REPORT_SEVERITY]:
- self.report[host][self.REPORT_GLOBAL] = {self.REPORT_SEVERITY: r.severity,
- self.REPORT_THREAT: r.threat}
+ for host, host_report in self.imported_oids.items():
+ self.oids[host], self.report[host] = dict(), dict()
- # Evaluate Global max severity
- if r.severity > self.report[self.REPORT_DEPLOYMENT][self.REPORT_SEVERITY]:
- self.report[self.REPORT_DEPLOYMENT] = {self.REPORT_SEVERITY: r.severity,
- self.REPORT_THREAT: r.threat}
+ # Create Pandas DataFrame from GreenBone report
+ vulns = pd.DataFrame(host_report)
- def classify_reports(self) -> None: ######################################<<<<<<<<<<<<<<<<<<########
+ # Add Label column
+ vulns[self.LABEL_COLUMN] = self.LABEL_NEW_VULNS
- # Init global aggregated variables
- self.init_glob_vars()
+ # Remove not important Vulnerabilties
+ vulns = vulns[vulns.severity >= self.SEVERITY_THR]
- for host, host_report in self.imported_oids.items():
- # Init aggregated variables per host
- self.init_host_vars(host)
- for res_report in host_report:
- logging.debug(res_report)
-
- # Skip if oid is not relevant
- if res_report.severity < self.SEVERITY_THR:
- self.update_summary(host, res_report)
- logging.debug("LOW SEVERITY -> SKIPPED")
- continue
-
- # Classify oid
- if not self.is_os and res_report.oid in self.os_all_oids:
- self.oids[host][self.OID_OS] += [res_report]
- logging.debug("OS RELATED")
- elif res_report.oid in self.accepted_oids:
- self.oids[host][self.OID_ACCEPTED].append(res_report)
- self.update_summary(host, res_report)
- logging.debug("ACCEPTED")
- elif res_report.oid in self.known_oids:
- self.oids[host][self.OID_DROPPED].append(res_report)
- logging.debug("DROPPED")
- else:
- self.oids[host][self.OID_NEW].append(res_report)
- self.update_summary(host, res_report)
- logging.debug("NEW")
+ # Label Acknowledged Vulnerabilities
+ vulns.loc[vulns.oids.isin(to_solve.oids),
+ self.LABEL_COLUMN] = self.LABEL_ACKNOWLEDGED_VULNS
+
+ # Label Excluded Vulnerabilities
+ vulns.loc[vulns.oids.isin(to_exclude.oids),
+ self.LABEL_COLUMN] = self.LABEL_REJECTED_VULNS
+
+ if not self.is_os:
+ # Label Os Vulnerabilities
+ vulns.loc[vulns.oids.isin(os_vulns.oids),
+ self.LABEL_COLUMN] = self.LABEL_OS_RELATED_VULNS
- for host,data in self.oids.items():
- for k,v_list in data.items():
- for o in v_list:
- logging.debug((host,k,o.oid,o.severity,o.threat,o.port))
+ # Collect Os Vulnerability oids
+ self.oids[host][self.LABEL_OS_RELATED_VULNS] = \
+ vulns[vulns.label == self.LABEL_OS_RELATED_VULNS].oids.to_list()
+
+ # Collect Acknowledged Vulnerability oids
+ self.oids[host][self.LABEL_ACKNOWLEDGED_VULNS] = \
+ vulns[vulns.label == self.LABEL_ACKNOWLEDGED_VULNS].oids.to_list()
+
+ # Collect Rejected Vulnerability oids
+ self.oids[host][self.LABEL_REJECTED_VULNS] = \
+ vulns[vulns.label == self.LABEL_REJECTED_VULNS].oids.to_list()
+
+ # Collect New Vulnerability oids
+ self.oids[host][self.LABEL_NEW_VULNS] = \
+ vulns[vulns.label == self.LABEL_NEW_VULNS].oids.to_list()
+
+ # Collect Acknowledged and New Vulnerabilities to create To-Solve Dataframe
+ to_solve = vulns[vulns[self.LABEL_COLUMN].isin(TO_SOLVE_VULNS)]
+
+ # Extract Max Severity per "ports" parameter
+ for ports, sev in to_solve.groupby(self.REPORT_PORTS).severity.max().items():
+ self.report[host][ports] = {self.REPORT_SEVERITY: sev}
+
+ # Compute Host Max Severity
+ max_severity = to_solve.severity.max()
+ self.report[host][self.REPORT_GLOBAL] = {self.REPORT_SEVERITY: max_severity}
+
+ # Check if Host Max Severity is greater the Deployment Max Severity
+ if max_severity > self.report[self.REPORT_DEPLOYMENT][self.REPORT_SEVERITY]:
+ self.report[self.REPORT_DEPLOYMENT][self.REPORT_SEVERITY] = max_severity
+
+ logging.debug(f"HOST: {host}")
+ logging.debug(f"\n{vulns}")
- # Extract global estimation
+ # Check if the Deployment Max Severity whether relevent or not
if self.report[self.REPORT_DEPLOYMENT][self.REPORT_SEVERITY] < self.SEVERITY_THR:
self.report[self.REPORT_GLOBAL] = self.MSG_OK
else:
@@ -748,12 +752,7 @@ class ReportManager():
return pretty_json(self.report)
def get_classified_oids(self) -> str:
- json_oids = dict()
- for host, data in self.oids.items():
- json_oids[host] = dict()
- for key, oids in data.items():
- json_oids[host][key] = [str(o) for o in oids]
- return pretty_json(json_oids)
+ return pretty_json(self.oids)
def create_msg(self, r: ResultReport):
msg = f" Detected oid: {r.oid}, severity: {r.severity}"
diff --git a/files/scan.py b/files/scan.py
index 1d9a6f6..d3d87f7 100644
--- a/files/scan.py
+++ b/files/scan.py
@@ -21,8 +21,6 @@ SUMMARY_FILENAME = "summary-report.json"
OIDS_FILENAME = "oids.json"
LOG_FILENAME = "log_scan.log"
-
-
parser = argparse.ArgumentParser(
description='Scan endpoints and machines')
--
GitLab
From cc976db01976f39c04a7b49ed5e2393170df4f0d Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:26:18 +0200
Subject: [PATCH 09/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 1d25fb3..bf5263a 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -670,9 +670,12 @@ class ReportManager():
self.imported_oids[host] = report
def show_imported_reports(self) -> None:
- logging.debug(json.dumps(self.imported_oids))
+ logging.debug("IMPORTED REPORTS")
+ for host, report in self.imported_oids:
+ logging.debug(f"HOST: {host}")
+ logging.debug(f"\n{pd.DataFrame(report)}")
+ logging.debug("")
-
def classify_reports(self) -> None:
TO_SOLVE_VULNS = [self.LABEL_NEW_VULNS,self.LABEL_ACKNOWLEDGED_VULNS]
@@ -754,11 +757,6 @@ class ReportManager():
def get_classified_oids(self) -> str:
return pretty_json(self.oids)
- def create_msg(self, r: ResultReport):
- msg = f" Detected oid: {r.oid}, severity: {r.severity}"
- msg += f", threat: {r.threat} and port: {r.port}\n"
- return msg
-
def write_data(self,
summary_filename: str,
oids_filename: str):
--
GitLab
From 286c5208c31e30bdda0b29fcff9d4d10d4cde378 Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:26:53 +0200
Subject: [PATCH 10/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index bf5263a..0a78248 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -671,7 +671,7 @@ class ReportManager():
def show_imported_reports(self) -> None:
logging.debug("IMPORTED REPORTS")
- for host, report in self.imported_oids:
+ for host, report in self.imported_oids.items():
logging.debug(f"HOST: {host}")
logging.debug(f"\n{pd.DataFrame(report)}")
logging.debug("")
--
GitLab
From bf386cd8016dc4f428e0adcccdad7c9d9f5a8edf Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:28:03 +0200
Subject: [PATCH 11/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 0a78248..b02ae05 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -677,6 +677,8 @@ class ReportManager():
logging.debug("")
def classify_reports(self) -> None:
+ logging.debug("\n\nCLASSIFIED OIDS")
+
TO_SOLVE_VULNS = [self.LABEL_NEW_VULNS,self.LABEL_ACKNOWLEDGED_VULNS]
to_solve = pd.Series({"oids": self.accepted_oids})
--
GitLab
From 715f10005b16ef9328f0faa431694bc6b483347d Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:40:47 +0200
Subject: [PATCH 12/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index b02ae05..200e8aa 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -614,7 +614,8 @@ class ReportManager():
self.os_all_oids = []
else:
self.os_oids = os_oids
- try:
+ self.os_all_oids = os_oids[self.os_name]
+ """try:
self.os_all_oids = tuple(set(reduce(lambda x,y: x + y,
os_oids.values())))
logging.debug("Imported os security oids")
@@ -622,7 +623,7 @@ class ReportManager():
except Exception as e:
logging.warning("Impossible extract oids from imported yaml")
- self.os_all_oids = []
+ self.os_all_oids = []"""
def extract_oids(self, lines: List[str]) -> Tuple[str]:
oids: List[str] = list()
@@ -670,7 +671,7 @@ class ReportManager():
self.imported_oids[host] = report
def show_imported_reports(self) -> None:
- logging.debug("IMPORTED REPORTS")
+ logging.debug("\nIMPORTED REPORTS")
for host, report in self.imported_oids.items():
logging.debug(f"HOST: {host}")
logging.debug(f"\n{pd.DataFrame(report)}")
--
GitLab
From fac79a9d8c67346c92a89530d54037575b9c17be Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:42:33 +0200
Subject: [PATCH 13/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index 200e8aa..ced0d39 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -615,6 +615,9 @@ class ReportManager():
else:
self.os_oids = os_oids
self.os_all_oids = os_oids[self.os_name]
+ logging.debug("Imported os security oids")
+ logging.debug(pretty_json(self.os_all_oids))
+
"""try:
self.os_all_oids = tuple(set(reduce(lambda x,y: x + y,
os_oids.values())))
--
GitLab
From 29ac69f85b697240abcafb5234ed7c09ab57debf Mon Sep 17 00:00:00 2001
From: Gioacchino Vino <gioacchino.vino@infn.it>
Date: Wed, 29 May 2024 17:47:49 +0200
Subject: [PATCH 14/14] Implement advanced vulnerability classification
---
files/gvm_library.py | 58 ++++++++++++++++----------------------------
1 file changed, 21 insertions(+), 37 deletions(-)
diff --git a/files/gvm_library.py b/files/gvm_library.py
index ced0d39..7d6b1d1 100644
--- a/files/gvm_library.py
+++ b/files/gvm_library.py
@@ -540,12 +540,6 @@ class ReportManager():
REPORT_THREAT = "threat"
REPORT_PORTS = "ports"
- # OIDS Classes
- OID_ACCEPTED = 'accepted-oids'
- OID_NEW = 'new-oids'
- OID_DROPPED = 'dropped-oids'
- OID_OS = 'os-related-oids'
-
# OS security repository configuration
OS_GIT_REPO = "baltig.infn.it/infn-cloud/os_security_checks.git"
OS_SEC_BRANCH = "new-oids"
@@ -561,8 +555,8 @@ class ReportManager():
SS_SEC_TOKEN = "GIT_SEC_TOKEN"
SS_SEC_DEST_DIR = "ss-repo"
SS_SEC_CHILD_DIR = "queues"
- SS_SEC_ACCEPTED_FILES = ['accepted.txt']
- SS_SEC_KNOWN_FILES = ['held.txt', 'new.txt', 'overridden.txt']
+ SS_SEC_ACKNOWLEDGED_OIDS_FILES = ['accepted.txt']
+ SS_SEC_REJECTED_OIDS_FILES = ['held.txt', 'new.txt', 'overridden.txt']
# Classification configuration
LABEL_COLUMN = "label"
@@ -614,19 +608,9 @@ class ReportManager():
self.os_all_oids = []
else:
self.os_oids = os_oids
- self.os_all_oids = os_oids[self.os_name]
+ self.os_all_oids = os_oids.get(self.os_name,[])
logging.debug("Imported os security oids")
logging.debug(pretty_json(self.os_all_oids))
-
- """try:
- self.os_all_oids = tuple(set(reduce(lambda x,y: x + y,
- os_oids.values())))
- logging.debug("Imported os security oids")
- logging.debug(pretty_json(self.os_all_oids))
-
- except Exception as e:
- logging.warning("Impossible extract oids from imported yaml")
- self.os_all_oids = []"""
def extract_oids(self, lines: List[str]) -> Tuple[str]:
oids: List[str] = list()
@@ -647,28 +631,28 @@ class ReportManager():
git.Repo.clone_from(repo_url, self.SS_SEC_DEST_DIR)
except Exception as e:
logging.warning(f"Impossible clone the ss scans repository, {e}")
- self.accepted_oids = []
- self.known_oids = []
+ self.acknowledged_oids = []
+ self.rejected_oids = []
else:
- accepted_oids: List[str] = []
- known_oids: List[str] = []
+ acknowledged_oids: List[str] = []
+ rejected_oids: List[str] = []
- for f in self.SS_SEC_ACCEPTED_FILES:
+ for f in self.SS_SEC_ACKNOWLEDGED_OIDS_FILES:
filename = os.path.join(files_dir,f)
with open(filename, 'r') as file:
- accepted_oids += self.extract_oids(file.readlines())
+ acknowledged_oids += self.extract_oids(file.readlines())
- for f in self.SS_SEC_KNOWN_FILES:
+ for f in self.SS_SEC_REJECTED_OIDS_FILES:
filename = os.path.join(files_dir,f)
with open(filename, 'r') as file:
- known_oids += self.extract_oids(file.readlines())
+ rejected_oids += self.extract_oids(file.readlines())
- self.accepted_oids = tuple(sorted(accepted_oids))
- self.known_oids = tuple(sorted(known_oids))
- logging.debug("accepted oids")
- logging.debug(pretty_json(self.accepted_oids))
- logging.debug("known oids")
- logging.debug(pretty_json(self.known_oids))
+ self.acknowledged_oids = tuple(sorted(acknowledged_oids))
+ self.rejected_oids = tuple(sorted(rejected_oids))
+ logging.debug("Acknowledged Oids")
+ logging.debug(pretty_json(self.acknowledged_oids))
+ logging.debug("Rejected Oids")
+ logging.debug(pretty_json(self.rejected_oids))
def import_report(self, host: str, report: Dict[str,Tuple]) -> None:
self.imported_oids[host] = report
@@ -685,8 +669,8 @@ class ReportManager():
TO_SOLVE_VULNS = [self.LABEL_NEW_VULNS,self.LABEL_ACKNOWLEDGED_VULNS]
- to_solve = pd.Series({"oids": self.accepted_oids})
- to_exclude = pd.Series({"oids": self.known_oids})
+ to_solve = pd.Series({"oids": self.acknowledged_oids})
+ to_exclude = pd.Series({"oids": self.rejected_oids})
os_vulns = pd.Series({"oids": self.os_all_oids})
self.report, self.oids = dict(), dict()
@@ -784,8 +768,8 @@ class ReportManager():
# Overwrite the detected oids to the host oids
self.os_oids[self.os_name] = []
for _ , data in self.oids.items():
- self.os_oids[self.os_name] += [a.oid for a in data[self.OID_ACCEPTED]]
- self.os_oids[self.os_name] += [n.oid for n in data[self.OID_NEW]]
+ self.os_oids[self.os_name] += [a.oid for a in data[self.LABEL_ACKNOWLEDGED_VULNS]]
+ self.os_oids[self.os_name] += [n.oid for n in data[self.LABEL_NEW_VULNS]]
with open(self.os_file, 'w') as f:
yaml.dump(self.os_oids, f)
--
GitLab