"""documentation_builder.py
Build the documentation based on sphinx and the read_the_doc layout.
License BSD-3-Clause
Copyright (c) 2021, New York University and Max Planck Gesellschaft.
"""
import argparse
import subprocess
import shutil
import fnmatch
import textwrap
import os
import typing
from pathlib import Path
import mpi_cmake_modules
from mpi_cmake_modules.utils import which
PathLike = typing.Union[str, os.PathLike]
def _get_cpp_file_patterns() -> typing.List[str]:
return ["*.h", "*.hh", "*.hpp", "*.hxx", "*.cpp", "*.c", "*.cc"]
def _find_doxygen() -> str:
"""Find the full path to the doxygen executable.
Raises:
Exception: if the doxygen executable is not found.
Returns:
The full path to the doxygen executable.
"""
exec_path = which("doxygen")
if exec_path is not None:
return exec_path
raise Exception(
"doxygen executable not found. You may try "
"'(sudo ) apt install doxygen*'"
)
def _find_breathe_apidoc() -> str:
"""Find the full path to the breathe-apidoc executable.
Raises:
Exception: if the breathe-apidoc executable is not found.
Returns:
The full path to the black executable.
"""
exec_path = which("breathe-apidoc")
if exec_path is not None:
return exec_path
raise Exception(
"breathe-apidoc executable not found. You may try "
"'(sudo -H) pip3 install breathe'"
)
def _find_sphinx_apidoc() -> str:
"""Find the full path to the sphinx-apidoc executable.
Raises:
Exception: if the sphinx-apidoc executable is not found.
Returns:
The full path to the black executable.
"""
exec_path = which("sphinx-apidoc")
if exec_path is not None:
return exec_path
raise Exception(
"sphinx-apidoc executable not found. You may try "
"'(sudo -H) pip3 install sphinx'"
)
def _find_sphinx_build() -> str:
"""Find the full path to the sphinx-build executable.
Raises:
Exception: if the sphinx-build executable is not found.
Returns:
The full path to the black executable.
"""
exec_path = which("sphinx-build")
if exec_path is not None:
return exec_path
raise Exception(
"sphinx-build executable not found. You may try "
"'(sudo -H) pip3 install sphinx'"
)
def _resource_path(project_source_dir: Path) -> Path:
"""
Fetch the resources path. This will contains all the configuration files
for the different executables: Doxyfile, conf.py, etc.
Args:
project_source_dir (str): Path to the source file of the project.
Raises:
Exception: if the resources folder is not found.
Returns:
pathlib.Path: Path to the configuration files.
"""
assert project_source_dir.is_dir()
# Find the resources from the package.
project_name = project_source_dir.name
if project_name == "mpi_cmake_modules":
resource_path = project_source_dir / "resources"
if not resource_path.is_dir():
raise Exception(
"failed to find the resource directory in "
+ str(mpi_cmake_modules.__path__)
)
return resource_path
# Find the resources from the installation of this package.
for module_path in mpi_cmake_modules.__path__:
resource_path = Path(module_path) / "resources"
if resource_path.is_dir():
return resource_path
raise Exception(
"failed to find the resource directory in "
+ str(mpi_cmake_modules.__path__)
)
def _build_doxygen_xml(doc_build_dir: Path, project_source_dir: Path):
"""
Use doxygen to parse the C++ source files and generate a corresponding xml
entry.
Args:
doc_build_dir (str): Path to where the doc should be built
project_source_dir (str): Path to the source file of the project.
"""
# Get project_name
project_name = project_source_dir.name
# Get the doxygen executable.
doxygen = _find_doxygen()
# Get the resources path.
resource_path = _resource_path(project_source_dir)
# get the Doxyfile.in file
doxyfile_in = resource_path / "sphinx" / "doxygen" / "Doxyfile.in"
assert doxyfile_in.is_file()
# Which files are going to be parsed.
doxygen_file_patterns = " ".join(_get_cpp_file_patterns())
# Where to put the doxygen output.
doxygen_output = doc_build_dir / "doxygen"
# Parse the Doxyfile.in and replace the value between '@'
with open(doxyfile_in, "rt") as f:
doxyfile_out_text = (
f.read()
.replace("@PROJECT_NAME@", project_name)
.replace("@PROJECT_SOURCE_DIR@", os.fspath(project_source_dir))
.replace("@DOXYGEN_FILE_PATTERNS@", doxygen_file_patterns)
.replace("@DOXYGEN_OUTPUT@", str(doxygen_output))
)
doxyfile_out = doxygen_output / "Doxyfile"
doxyfile_out.parent.mkdir(parents=True, exist_ok=True)
with open(doxyfile_out, "wt") as f:
f.write(doxyfile_out_text)
bashCommand = doxygen + " " + str(doxyfile_out)
process = subprocess.Popen(
bashCommand.split(), stdout=subprocess.PIPE, cwd=str(doxygen_output)
)
output, error = process.communicate()
print("Doxygen output:\n", output.decode("UTF-8"))
print("Doxygen error:\n", error)
print("")
def _build_breath_api_doc(doc_build_dir: Path):
"""
Use breathe_apidoc to parse the xml output from Doxygen and generate
'.rst' files.
Args:
doc_build_dir (str): Path where to create the temporary output.
"""
breathe_apidoc = _find_breathe_apidoc()
breathe_input = doc_build_dir / "doxygen" / "xml"
breathe_output = doc_build_dir / "breathe_apidoc"
breathe_option = "-f -g class,interface,struct,union,file,namespace,group"
bashCommand = (
breathe_apidoc
+ " -o "
+ str(breathe_output)
+ " "
+ str(breathe_input)
+ " "
+ str(breathe_option)
)
process = subprocess.Popen(
bashCommand.split(), stdout=subprocess.PIPE, cwd=str(doc_build_dir)
)
output, error = process.communicate()
print("breathe-apidoc output:\n", output.decode("UTF-8"))
print("breathe-apidoc error:\n", error)
print("")
def _build_sphinx_api_doc(doc_build_dir: Path, python_source_dir: Path):
"""
Use sphinx_apidoc to parse the python files output from Doxygen and
generate '.rst' files.
Args:
doc_build_dir (str): Path where to create the temporary output.
project_source_dir (str): Path to the source file of the project.
"""
# define input folder
if python_source_dir.is_dir():
sphinx_apidoc = _find_sphinx_apidoc()
sphinx_apidoc_input = str(python_source_dir)
sphinx_apidoc_output = str(doc_build_dir)
bashCommand = (
sphinx_apidoc
+ " --separate "
+ " -o "
+ sphinx_apidoc_output
+ " "
+ sphinx_apidoc_input
)
process = subprocess.Popen(
bashCommand.split(), stdout=subprocess.PIPE, cwd=str(doc_build_dir)
)
output, error = process.communicate()
print("sphinx-apidoc output:\n", output.decode("UTF-8"))
print("sphinx-apidoc error:\n", error)
else:
print("No python module for sphinx-apidoc to parse.")
print("")
def _build_sphinx_build(doc_build_dir: Path):
"""
Use sphinx_build to parse the cmake and rst files previously generated and
generate the final html layout.
Args:
doc_build_dir (str): Path where to create the temporary output.
"""
sphinx_build = _find_sphinx_build()
bashCommand = (
sphinx_build
+ " -M html "
+ str(doc_build_dir)
+ " "
+ str(doc_build_dir)
)
process = subprocess.Popen(
bashCommand.split(), stdout=subprocess.PIPE, cwd=str(doc_build_dir)
)
output, error = process.communicate()
print("sphinx-apidoc output:\n", output.decode("UTF-8"))
print("sphinx-apidoc error:\n", error)
def _search_for_cpp_api(
doc_build_dir: Path, project_source_dir: Path, resource_dir: Path
) -> str:
"""Search if there is a C++ api do document, and document it.
Args:
doc_build_dir (str): Path where to create the temporary output.
project_source_dir (str): Path to the source file of the project.
resource_dir (str): Path to the resources files for the build.
Returns:
str: String added to the main index.rst in case there is a C++ api.
"""
cpp_api = ""
# Search for C++ files
has_cpp = False
for p in project_source_dir.glob("**/*"):
if any(
fnmatch.fnmatch(str(p), pattern)
for pattern in _get_cpp_file_patterns()
):
has_cpp = True
break
if has_cpp:
print("Found C++ files, add C++ API documentation")
# Introduce this toc tree in the main index.rst
cpp_api = textwrap.dedent(
"""
.. toctree::
:caption: C++ API
:maxdepth: 2
doxygen_index
"""
)
# Copy the index of the C++ API.
shutil.copy(
resource_dir
/ "sphinx"
/ "sphinx"
/ "doxygen_index_one_page.rst.in",
doc_build_dir / "doxygen_index_one_page.rst",
)
shutil.copy(
resource_dir / "sphinx" / "sphinx" / "doxygen_index.rst.in",
doc_build_dir / "doxygen_index.rst",
)
# Build the doxygen xml files.
_build_doxygen_xml(doc_build_dir, project_source_dir)
# Generate the .rst corresponding to the doxygen xml
_build_breath_api_doc(doc_build_dir)
else:
print("No C++ files found.")
return cpp_api
def _search_for_python_api(
doc_build_dir: Path,
project_source_dir: Path,
package_path: typing.Optional[Path] = None,
) -> str:
"""Search for a Python API and build it's documentation.
Args:
doc_build_dir (str): Path where to create the temporary output.
project_source_dir (str): Path to the source file of the project.
Returns:
str: String added to the main index.rst in case there is a Python api.
"""
python_api = ""
# Get the project name form the source path.
project_name = project_source_dir.name
if package_path is None:
package_path_candidates = [
project_source_dir / project_name,
project_source_dir / "python" / project_name,
project_source_dir / "src" / project_name,
]
for p in package_path_candidates:
if p.is_dir():
package_path = p
break
# Search for Python API.
if package_path:
# Introduce this toc tree in the main index.rst
python_api = textwrap.dedent(
"""
.. toctree::
:caption: Python API
:maxdepth: 3
modules
* :ref:`modindex`
"""
)
_build_sphinx_api_doc(doc_build_dir, package_path)
return python_api
def _search_for_cmake_api(
doc_build_dir: Path, project_source_dir: Path, resource_dir: Path
) -> str:
cmake_api = ""
# Search for CMake API.
cmake_files = [
p.resolve()
for p in project_source_dir.glob("cmake/*")
if p.suffix in [".cmake"] or p.name == "CMakeLists.txt"
]
if cmake_files:
# Introduce this toc tree in the main index.rst
cmake_api = textwrap.dedent(
"""
.. toctree::
:caption: CMake API
:maxdepth: 3
cmake_doc
"""
)
doc_cmake_module = ""
for cmake_file in cmake_files:
doc_cmake_module += cmake_file.stem + "\n"
doc_cmake_module += len(cmake_file.stem) * "-" + "\n\n"
doc_cmake_module += (
".. cmake-module:: cmake/" + cmake_file.name + "\n\n"
)
with open(
resource_dir / "sphinx" / "sphinx" / "cmake_doc.rst.in", "rt"
) as f:
out_text = f.read().replace("@DOC_CMAKE_MODULE@", doc_cmake_module)
with open(str(doc_build_dir / "cmake_doc.rst"), "wt") as f:
f.write(out_text)
shutil.copytree(
project_source_dir / "cmake",
doc_build_dir / "cmake",
)
return cmake_api
def _search_for_general_documentation(
doc_build_dir: Path, project_source_dir: Path, resource_dir: Path
) -> str:
general_documentation = ""
doc_path_candidates = [
project_source_dir / "doc",
project_source_dir / "docs",
]
doc_path = None
for p in doc_path_candidates:
if p.is_dir():
doc_path = p
break
# Search for additional doc.
if doc_path:
general_documentation = textwrap.dedent(
"""
.. toctree::
:caption: General Documentation
:maxdepth: 2
general_documentation
"""
)
shutil.copy(
resource_dir
/ "sphinx"
/ "sphinx"
/ "general_documentation.rst.in",
doc_build_dir / "general_documentation.rst",
)
shutil.copytree(
doc_path,
doc_build_dir / "doc",
)
return general_documentation
[docs]def build_documentation(
build_dir: PathLike,
project_source_dir: PathLike,
project_version,
python_pkg_path: typing.Optional[PathLike] = None,
):
# make sure all paths are of type Path
doc_build_dir = Path(build_dir)
project_source_dir = Path(project_source_dir)
if python_pkg_path is not None:
python_pkg_path = Path(python_pkg_path)
#
# Initialize the paths
#
# Get the project name form the source path.
project_name = project_source_dir.name
# Create the folder architecture inside the build folder.
shutil.rmtree(doc_build_dir, ignore_errors=True)
doc_build_dir.mkdir(parents=True, exist_ok=True)
# Get the path to resource files.
resource_dir = Path(_resource_path(project_source_dir))
#
# Parametrize the final doc layout depending we have CMake/Python/C++ api.
#
# String to replace in the main index.rst
cpp_api = _search_for_cpp_api(
doc_build_dir, project_source_dir, resource_dir
)
python_api = _search_for_python_api(
doc_build_dir, project_source_dir, python_pkg_path
)
cmake_api = _search_for_cmake_api(
doc_build_dir, project_source_dir, resource_dir
)
general_documentation = _search_for_general_documentation(
doc_build_dir, project_source_dir, resource_dir
)
#
# Configure the config.py and the index.rst.
#
# configure the index.rst.in.
header = "Welcome to " + project_name + "'s documentation!"
header += "\n" + len(header) * "=" + "\n"
with open(resource_dir / "sphinx" / "sphinx" / "index.rst.in", "rt") as f:
out_text = (
f.read()
.replace("@HEADER@", header)
.replace("@GENERAL_DOCUMENTATION@", general_documentation)
.replace("@CPP_API@", cpp_api)
.replace("@PYTHON_API@", python_api)
.replace("@CMAKE_API@", cmake_api)
)
with open(doc_build_dir / "index.rst", "wt") as f:
f.write(out_text)
# configure the config.py.in.
with open(resource_dir / "sphinx" / "sphinx" / "conf.py.in", "rt") as f:
out_text = (
f.read()
.replace("@PROJECT_SOURCE_DIR@", os.fspath(project_source_dir))
.replace("@PROJECT_NAME@", project_name)
.replace("@PROJECT_VERSION@", project_version)
.replace(
"@DOXYGEN_XML_OUTPUT@", str(doc_build_dir / "doxygen" / "xml")
)
)
with open(doc_build_dir / "conf.py", "wt") as f:
f.write(out_text)
#
# Copy the license and readme file.
#
readme = [
p.resolve()
for p in project_source_dir.glob("*")
if p.name.lower() in ["readme.md", "readme.rst"]
]
# sort alphabetically so that "readme.md" is preferred in case both are
# found
readme = sorted(readme)
if readme:
shutil.copy(readme[0], doc_build_dir / "readme.md")
license_file = [
p.resolve()
for p in project_source_dir.glob("*")
if p.name in ["LICENSE", "license.txt"]
]
if license_file:
shutil.copy(license_file[0], doc_build_dir / "license.txt")
#
# Generate the html doc
#
_build_sphinx_build(doc_build_dir)
[docs]def main():
def AbsolutePath(path):
return Path(path).absolute()
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--output-dir",
required=True,
type=AbsolutePath,
help="Build directory",
)
parser.add_argument(
"--package-dir",
required=True,
type=AbsolutePath,
help="Package directory",
)
parser.add_argument(
"--python-dir",
type=AbsolutePath,
help="""Directory containing the Python package. If not set, it is
auto-detected inside the package directory
""",
)
parser.add_argument(
"--project-version", required=True, type=str, help="Package version"
)
parser.add_argument(
"--force",
"-f",
action="store_true",
help="Do not ask before deleting files.",
)
args = parser.parse_args()
if not args.force and args.output_dir.exists():
print(
"Output directory {} already exists."
" It will be deleted if you proceed!".format(args.output_dir)
)
c = input("Continue? [y/N] ")
if c not in ["y", "Y", "yes"]:
print("Abort.")
return
build_documentation(
args.output_dir,
args.package_dir,
args.project_version,
python_pkg_path=args.python_dir,
)
if __name__ == "__main__":
main()