Merge pull request #54 from TuriJ95/issue-52

fixed extra requirements

Author: FredHappyface

licensecheck/get_deps.py

@@ -2,6 +2,7 @@
 """
 from __future__ import annotations
 
+import re
 from importlib import metadata
 from pathlib import Path
 from typing import Any
@@ -56,8 +57,10 @@ def _doGetReqs(
 	using: str, extras: str | None, pyproject: dict[str, Any], requirementsPaths: list[Path]
 ) -> set[ucstr]:
 	resolveReq = lambda req: ucstr(pkg_resources.Requirement.parse(req).project_name)
+	resolveExtraReq = lambda extra_req: re.sub("extra == ", "", re.findall(r"extra == '.*?'",extra_req)[0].replace("'", "")) if len(re.findall(r"extra == '.*?'",extra_req)) > 0 else None
 
 	reqs = set()
+	extras_reqs = dict()
 
 	if using == "poetry":
 		try:
@@ -75,8 +78,12 @@ def _doGetReqs(
 			reqLists.append(project.get("dev-dependencies", {}))
 		for reqList in reqLists:
 			for req in reqList:
-				reqs.add(resolveReq(req))
-
+				try:
+					extra = re.search(r'(?<=\[)(.*?)(?=\])',req).group(0)
+					extras_reqs[resolveReq(req)] = extra
+					reqs.add(f"{resolveReq(req)}[{extra}]")
+				except (KeyError, AttributeError):
+					reqs.add(resolveReq(req))
 	# PEP631 (hatch)
 	if using == "PEP631":
 		try:
@@ -90,7 +97,12 @@ def _doGetReqs(
 			reqLists.extend(project["optional-dependencies"][x] for x in extras.split(";"))
 		for reqList in reqLists:
 			for req in reqList:
-				reqs.add(resolveReq(req))
+				try:
+					extra = re.search(r'(?<=\[)(.*?)(?=\])',req).group(0)
+					extras_reqs[resolveReq(req)] = extra
+					reqs.add(f"{resolveReq(req)}[{extra}]")
+				except (KeyError, AttributeError):
+					reqs.add(resolveReq(req))
 
 	# Requirements
 	if using == "requirements":
@@ -100,7 +112,12 @@ def _doGetReqs(
 
 			with open(reqPath, encoding="utf-8") as requirementsTxt:
 				for req in requirements.parse(requirementsTxt):
-					reqs.add(str(req.name).lower())
+					if len(req.extras)>0:
+						extras_reqs[resolveReq(req.name)] = req.extras
+						for extra in req.extras:
+							reqs.add(f"{resolveReq(req.name)}[{extra}]")
+					else:
+						reqs.add(resolveReq(req.name))
 
 	try:
 		reqs.remove("PYTHON")
@@ -115,16 +132,19 @@ def _doGetReqs(
 			for req in [resolveReq(req) for req in pkgMetadata.get_all("Requires-Dist") or []]:
 				requirementsWithDeps.add(req)
 		except metadata.PackageNotFoundError:
-			request = session.get(f"https://pypi.org/pypi/{requirement}/json", timeout=60)
+			request = session.get(f"https://pypi.org/pypi/{requirement.split('[')[0]}/json", timeout=60)
 			response = request.json()
 			try:
-				for req in [resolveReq(req) for req in response["info"]["requires_dist"]]:
-					requirementsWithDeps.add(req)
+				for dependency in response["info"]["requires_dist"]:
+					if resolveExtraReq(dependency) is not None:
+						if (resolveReq(requirement) in extras_reqs.keys()) and (resolveExtraReq(dependency) in extras_reqs[resolveReq(requirement)]):
+							requirementsWithDeps.add(resolveReq(dependency))
+					else:
+						requirementsWithDeps.add(resolveReq(dependency))
 			except (KeyError, TypeError):
 				pass
 
-	return requirementsWithDeps
-
+	return {r.split('[')[0] for r in requirementsWithDeps}
 
 def getDepsWithLicenses(
 	using: str,

tests/data/test_requirements.txt

@@ -0,0 +1,13 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --no-emit-index-url --output-file=requirements/tmp.txt subset.in
+#
+
+pandas[excel]==2.0.3
+
+-c requirements_dev.txt
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

tests/test_get_deps.py

@@ -7,58 +7,55 @@
 THISDIR = Path(__file__).resolve().parent
 
 
-def test_doGetReqs():
+def test_doGetReqs_PEP631():
 
 	using = "PEP631"
 	extras = "socks"
 	pyproject = tomli.loads((THISDIR / "data/pep631_socks.toml").read_text(encoding="utf-8"))
 	requirementsPaths = []
 
 	assert get_deps._doGetReqs(using, extras, pyproject, requirementsPaths) == {
-		"ATTRS",
-		"BACKPORTS.SHUTIL-GET-TERMINAL-SIZE",
-		"BACKPORTS.SSL-MATCH-HOSTNAME",
-		"CACHED-PROPERTY",
-		"CERTIFI",
-		"CHARDET",
-		"CHARSET-NORMALIZER",
-		"CLICK",
-		"COLORAMA",
-		"DISTRO",
-		"DOCKER",
-		"DOCKERPTY",
-		"DOCOPT",
-		"ENUM34",
-		"FQDN",
-		"IDNA",
-		"IMPORTLIB-RESOURCES",
-		"IPADDRESS",
-		"ISODURATION",
-		"JSONPOINTER",
-		"JSONSCHEMA",
-		"JSONSCHEMA-SPECIFICATIONS",
-		"PACKAGING",
-		"PARAMIKO",
-		"PKGUTIL-RESOLVE-NAME",
-		"PYSOCKS",
-		"PYTHON-DOTENV",
-		"PYTHON-SOCKS",
-		"PYWIN32",
-		"PYYAML",
-		"REFERENCING",
-		"REQUESTS",
-		"RFC3339-VALIDATOR",
-		"RFC3986-VALIDATOR",
-		"RFC3987",
-		"RPDS-PY",
-		"SPHINX",
-		"SPHINX-RTD-THEME",
-		"SUBPROCESS32",
-		"TEXTTABLE",
-		"URI-TEMPLATE",
-		"URLLIB3",
-		"WEBCOLORS",
-		"WEBSOCKET-CLIENT",
-		"WEBSOCKETS",
-		"WSACCEL",
+		'DOCKERPTY',
+		'PACKAGING',
+		'ATTRS',
+		'JSONSCHEMA',
+		'PYYAML',
+		'PYSOCKS',
+		'CERTIFI',
+		'ENUM34',
+		'DOCKER',
+		'TEXTTABLE',
+		'PYWIN32',
+		'JSONSCHEMA-SPECIFICATIONS',
+		'IPADDRESS',
+		'PKGUTIL-RESOLVE-NAME',
+		'DOCOPT',
+		'BACKPORTS.SSL-MATCH-HOSTNAME',
+		'PARAMIKO',
+		'IDNA',
+		'COLORAMA',
+		'IMPORTLIB-RESOURCES',
+		'CACHED-PROPERTY',
+		'DISTRO',
+		'BACKPORTS.SHUTIL-GET-TERMINAL-SIZE',
+		'CHARSET-NORMALIZER',
+		'URLLIB3',
+		'WEBSOCKET-CLIENT',
+		'RPDS-PY',
+		'SUBPROCESS32',
+		'REQUESTS',
+		'REFERENCING',
+		'CHARDET',
+		'PYTHON-DOTENV'
 	}
+
+def test_doGetReqs_requirements():
+
+	using = "requirements"
+	extras = f"{THISDIR}/data/test_requirements.txt"
+	pyproject = {}
+	requirementsPaths = [Path(f"{THISDIR}/data/test_requirements.txt")]
+	deps = get_deps._doGetReqs(using, extras, pyproject, requirementsPaths)
+	assert deps == {'NUMPY', 'ODFPY', 'OPENPYXL', 'PANDAS', 'PYTHON-DATEUTIL', 'PYTZ', 'PYXLSB', 'TZDATA', 'XLRD', 'XLSXWRITER'}
+	assert 'OPENPYXL' in deps
+	assert 'XARRAY' not in deps #xarray is an optional dependecy of pandas associated with 'computation' key that is not tracked in test_requirements.txt