Add checking of all endpoints defined in FTL but not in the OpenAPI specs and vice versa

Signed-off-by: DL6ER <dl6er@dl6er.de>

test/api/checkAPI.py

@@ -48,7 +48,7 @@ if __name__ == "__main__":
 	# Check if endpoints that are in both FTL and OpenAPI specs match
 	# and have the same response format. Also verify that the examples
 	# matches the OpenAPI specs.
-	print("Verifying the individual endpoint properties...")
+	print("Verifying the individual OpenAPI endpoint properties...")
 	teleporter = None
 	for path in openapi.endpoints["get"]:
 		# We do not check the action endpoints as they'd trigger
@@ -56,19 +56,32 @@ if __name__ == "__main__":
 		# gravity, stutting down the system, etc.
 		if path.startswith("/api/action"):
 			continue
-		verifyer = ResponseVerifyer(ftl, openapi)
-		errors = verifyer.verify_endpoint(path)
-		if verifyer.teleporter_archive is not None:
-			teleporter = verifyer.teleporter_archive
-		if len(errors) == 0:
-			print("  GET " + path + " (" + verifyer.auth_method + " auth): OK")
-		else:
-			print("  GET " + path + " (" + verifyer.auth_method + " auth):")
-			for error in errors:
-				print("  - " + error)
-			errs[2] += len(errors)
+		with ResponseVerifyer(ftl, openapi) as verifyer:
+			errors = verifyer.verify_endpoint(path)
+			if verifyer.teleporter_archive is not None:
+				teleporter = verifyer.teleporter_archive
+			if len(errors) == 0:
+				print("  GET " + path + " (" + verifyer.auth_method + " auth): OK")
+			else:
+				print("  GET " + path + " (" + verifyer.auth_method + " auth):")
+				for error in errors:
+					print("  - " + error)
+				errs[2] += len(errors)
 	print("")
 
+	# Verify that all the endpoint defined by GET /api/endpoints are documented
+	# and that there are no undocumented endpoints
+	print("Verifying the /api/endpoints endpoint...")
+	verifyer = ResponseVerifyer(ftl, openapi)
+	errors = verifyer.verify_endpoints()
+	if len(errors) == 0:
+		print("  GET /api/endpoints: OK")
+	else:
+		print("  Errors:")
+		for error in errors:
+			print("  - " + error)
+		errs[2] += len(errors)
+
 	# Verify FTL Teleporter import
 	print("Verifying FTL Teleporter import...")
 	verifyer = ResponseVerifyer(ftl, openapi)

test/api/libs/openAPI.py

@@ -15,7 +15,7 @@ import json
 class openApi():
 
 	# List of methods we want to extract
-	METHODS = ["get", "post", "put", "delete"]
+	METHODS = ["get", "post", "put", "patch", "delete"]
 
 	def __init__(self, base_path: str, api_root: str = "/api") -> None:
 		# Store arguments

test/api/libs/responseVerifyer.py

@@ -34,6 +34,14 @@ class ResponseVerifyer():
 		self.errors = []
 
 
+	def __enter__(self):
+		return self
+
+
+	def __exit__(self, exc_type, exc_value, traceback):
+		return
+
+
 	def flatten_dict(self, d: MutableMapping, parent_key: str = '', sep: str ='.') -> MutableMapping:
 		items = []
 		# Iterate over all items in the dictionary
@@ -67,7 +75,7 @@ class ResponseVerifyer():
 			return self.errors
 
 		# Get YAML response schema and examples (if applicable)
-		expected_mimetype = True
+		expected_mimetype = None
 		# Assign random authentication method so we can test them all
 		authentication_method = random.choice([a for a in AuthenticationMethods])
 		# Check if the expected response is defined in the API specs
@@ -86,6 +94,11 @@ class ResponseVerifyer():
 				authentication_method = AuthenticationMethods.HEADER
 				YAMLresponseSchema = None
 				YAMLresponseExamples = None
+			elif 'text/html' in content:
+				expected_mimetype = 'text/html'
+				jsonData = content[expected_mimetype]
+				YAMLresponseSchema = None
+				YAMLresponseExamples = None
 		else:
 			# No response defined
 			return self.errors
@@ -166,6 +179,17 @@ class ResponseVerifyer():
 
 				# Store Teleporter archive for later use
 				self.teleporter_archive = FTLresponse
+		elif expected_mimetype == "text/html":
+			# Decode the response if it is bytes
+			if type(FTLresponse) is bytes:
+				FTLresponse = FTLresponse.decode("utf-8")
+			elif type(FTLresponse) is not str:
+				self.errors.append("FTL's response is neither bytes nor string")
+			# Check if the document starts with either "<!DOCTYPE html>" or
+			# "<html>" (case-insensitive)
+			r = FTLresponse.lower()
+			if not r.startswith("<!doctype html>") and not r.startswith("<html>"):
+				self.errors.append("FTL's response does not start with <!DOCTYPE html> or <html>")
 		else:
 			self.errors.append("Checker script does not know how to check for mimetype \"" + expected_mimetype + "\"")
 
@@ -328,3 +352,53 @@ class ResponseVerifyer():
 				self.errors.append(f"FTL's reply ({str(ftl_type)}) does not match defined type ({yaml_type}) in {flat_path}")
 				return False
 		return all_okay
+
+
+	def verify_endpoints(self):
+		# Get FTL response
+		authentication_method = random.choice([a for a in AuthenticationMethods])
+		FTLresponse = self.ftl.GET("/api/endpoints", authenticate = authentication_method)
+		if FTLresponse is None:
+			self.errors.append("No response from FTL API")
+			return self.errors
+
+		# Construct full URI to check (this is what we specify in OpenAPI specs)
+		for method in FTLresponse['endpoints']:
+			for endpoint in FTLresponse['endpoints'][method]:
+				endpoint["full_uri"] = endpoint["uri"] + endpoint["parameters"] # type: str
+				# If the endpoint starts with /api, remove this part (it is not
+				# part of the YAML specs)
+				if endpoint["full_uri"].startswith("/api"):
+					endpoint["full_uri"] = endpoint["full_uri"][4:]
+
+		# Do the same for the specified endpoints in the OpenAPI specs
+		openapi = {}
+		for endpoint in self.openapi.paths:
+			openapi[endpoint] = {}
+			for method in self.openapi.paths[endpoint]:
+				if method not in self.openapi.METHODS:
+					# Skip keys like "parameters" and "summary"
+					continue
+				#if "parameters" in self.openapi.paths[endpoint][method]:
+				#	# Construct full URI to check (this is what we specify in OpenAPI specs)
+				#	for param in self.openapi.paths[endpoint][method]["parameters"]:
+				#		if param["in"] == "query":
+				#			endpoint += "?" + param["name"] + "=" + urllib.parse.quote_plus(str(param["example"]))
+				openapi[endpoint][method] = endpoint
+
+		# Check if FTL reports endpoints not defined in the API specs
+		for method in FTLresponse['endpoints']:
+			for endpoint in FTLresponse['endpoints'][method]:
+				m = method.upper() # type: str
+				if endpoint["full_uri"] not in self.openapi.paths or method not in self.openapi.paths[endpoint["full_uri"]]:
+					self.errors.append("Endpoint " + m + " " + endpoint["full_uri"] + " not found in the OpenAPI specs")
+
+		# Check if all endpoints defined in the API specs are also defined in FTL
+		for endpoint in openapi:
+			for method in openapi[endpoint]:
+				full_uris = [ep["full_uri"] for ep in FTLresponse['endpoints'][method]] # type: list[str]
+				if endpoint not in full_uris:
+					m = method.upper() # type: str
+					self.errors.append("Endpoint " + m + " " + endpoint + " not found in FTL endpoints")
+
+		return self.errors