Code review: contrib/gendocs.sh -> contrib/gendocs.py

What: migrate from shell and Scala to pure Python.

Why: to improve the UX as users likely already have Python installed.

3 files changed, 298 insertions, 208 deletions

diff --git a/contrib/gendocs.py b/contrib/gendocs.py
new file mode 100755
index 00000000..cf13a041
--- /dev/null
+++ b/contrib/gendocs.py
@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+
+# This script generates a static documentation web site
+# by parsing the `**/*.asiidoc` files from the repository.
+#
+# Dependencies:
+# * Relatively recent Python 3 - must support `typing.Optional`.
+# * `xdg-open` for opening the final result in a web browser.
+# * `antora` - a static documentation web site generator,
+#     see: https://antora.org
+#
+# Usage:
+# ```console
+# $ ./contrib/gendocs.py
+# ```
+#
+# It should open the generated web site in a browser.
+
+
+import atexit
+import glob
+import linksfixer
+import os
+import pathlib
+import shutil
+import subprocess
+
+# Get the script directory
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
+# Switch to the projects root dir
+os.chdir(os.path.join(script_dir, ".."))
+
+
+# Idempotency
+def cleanup():
+    try:
+        shutil.move("test/tests.asciidoc", "test/README.asciidoc")
+    except BaseException:
+        pass
+    try:
+        shutil.move("index.asciidoc", "README.asciidoc")
+    except BaseException:
+        pass
+    try:
+        shutil.move("VIMTOKAK.asciidoc", "VIMTOKAK")
+    except BaseException:
+        pass
+
+
+# Set up cleanup to be executed on script exit
+atexit.register(cleanup)
+
+# Antora does not like missing links
+shutil.rmtree("libexec", ignore_errors=True)
+
+# Canonical Antora paths
+os.makedirs("modules/ROOT/images", exist_ok=True)
+os.makedirs("modules/ROOT/pages", exist_ok=True)
+
+# Better naming
+shutil.move("test/README.asciidoc", "test/tests.asciidoc")
+
+# The main page
+shutil.move("README.asciidoc", "index.asciidoc")
+
+# We want this in the documentation as well
+shutil.move("VIMTOKAK", "VIMTOKAK.asciidoc")
+
+# Put necessary images to the Antora canonical directory
+for gif_file in glob.glob("doc/*.gif"):
+    shutil.copy(gif_file, "modules/ROOT/images/")
+
+# Create directories structure matching the project's original structure
+for asciidoc_file in glob.glob("**/*.asciidoc", recursive=True):
+    file_dir = os.path.dirname(asciidoc_file)
+    page_dir = os.path.join("modules/ROOT/pages", file_dir)
+    os.makedirs(page_dir, exist_ok=True)
+
+    # Take all `.asciidoc` files and put them into Antora's canonical directory
+    # with the canonical `.adoc` file extension
+    adoc = pathlib.Path(asciidoc_file).stem + ".adoc"
+    dest = os.path.join(page_dir, adoc)
+    shutil.copy(asciidoc_file, dest)
+
+# Fix images
+for adoc_file in glob.glob("modules/ROOT/pages/**/*.adoc", recursive=True):
+    with open(adoc_file, "r") as f:
+        content = f.read()
+    content = content.replace("image::doc/", "image::")
+    with open(adoc_file, "w") as f:
+        f.write(content)
+
+# Fix links using another Python script.
+for adoc in glob.glob("modules/ROOT/pages/**/*.adoc", recursive=True):
+    linksfixer.fix_file(adoc)
+
+
+# Crafted manually from all the files at the time of writing the script.
+# TODO: Generate automatically.
+nav_content = """
+* xref:index.adoc[Getting Started]
+* xref:doc/pages/commands.adoc[Commands]
+* xref:doc/pages/expansions.adoc[Expansions]
+* xref:doc/pages/execeval.adoc[Exec Eval]
+* xref:doc/pages/scopes.adoc[Scopes]
+* xref:doc/pages/faces.adoc[Faces]
+* xref:doc/pages/buffers.adoc[Buffers]
+* xref:doc/pages/registers.adoc[Registers]
+* xref:doc/pages/mapping.adoc[Mapping]
+* xref:doc/pages/hooks.adoc[Hooks]
+* xref:doc/pages/command-parsing.adoc[Command Parsing]
+* xref:doc/pages/keys.adoc[Keys]
+* xref:doc/pages/regex.adoc[Regex]
+* xref:doc/pages/options.adoc[Options]
+* xref:doc/pages/highlighters.adoc[Highlighters]
+* xref:doc/pages/modes.adoc[Modes]
+* xref:doc/pages/keymap.adoc[KEYMAP]
+* xref:doc/pages/faq.adoc[FAQ]
+* xref:doc/pages/changelog.adoc[Changelog]
+* xref:doc/design.adoc[Design]
+* xref:doc/coding-style.adoc[Coding Style]
+* xref:doc/writing_scripts.adoc[Writing Scripts]
+* xref:doc/json_ui.adoc[JSON UI]
+* xref:doc/autoedit.adoc[Autoedit]
+* xref:doc/interfacing.adoc[Interfacing]
+* xref:rc/tools/lint.adoc[Linting]
+* xref:rc/tools/autorestore.adoc[Autorestore]
+* xref:rc/tools/doc.adoc[Doc]
+* xref:test/tests.adoc[Tests]
+* xref:VIMTOKAK.adoc[Vi(m) to Kakoune]
+"""
+
+with open("modules/ROOT/nav.adoc", "w") as f:
+    f.write(nav_content)
+
+# Antora's component description file
+antora_yml_content = """
+name: Kakoune
+nav:
+  - modules/ROOT/nav.adoc
+title: Kakoune
+version: latest
+"""
+
+with open("antora.yml", "w") as f:
+    f.write(antora_yml_content)
+
+# Antora playbook file
+antora_playbook_content = """
+asciidoc:
+  attributes:
+
+    # Do not complain on missing attributes,
+    # TODO: fix and turn to a fatal warning
+    attribute-missing: skip
+
+    # To fix links
+    idprefix: ""
+
+    # To fix links
+    idseparator: "-"
+
+    # Better to be reproducible, in general
+    reproducible: true
+
+    # More convenient to turn sections to IDs
+    sectids: true
+
+    # More convenient to have sections as links
+    sectlinks: true
+
+    # Do not want to miss something
+    sectnumlevels: 5
+
+    # More convenient to number the sections
+    sectnums: true
+
+    # Do not want to miss something
+    toclevels: 5
+
+  sourcemap: true
+
+content:
+  sources:
+  - url: ./
+    branches: ["HEAD"]
+
+runtime:
+  cache_dir: ./cache # More convenient for the development
+  fetch: true # More convenient for the development
+
+  log:
+    failure_level: fatal
+    level: warn
+
+output:
+  clean: true # More convenient for the development
+  dir: ./build # Simpler to have it explicit in code
+
+site:
+  title: Kakoune Docs
+
+ui:
+  bundle:
+    url: https://gitlab.com/antora/antora-ui-default/-/jobs/artifacts/HEAD/raw/build/ui-bundle.zip?job=bundle-stable
+"""
+
+with open("antora-playbook.yml", "w") as f:
+    f.write(antora_playbook_content)
+
+# Finally, generate the documentation,
+# will saved to the output directory specified in the antora-playbook.yml
+subprocess.run(["antora", "generate", "antora-playbook.yml"])
+subprocess.run(["xdg-open", "./build/Kakoune/latest/index.html"])
diff --git a/contrib/gendocs.sh b/contrib/gendocs.sh
deleted file mode 100755
index f271a1a3..00000000
--- a/contrib/gendocs.sh
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env bash
-
-# This script searches for `**/*.asciidoc` files in the repository,
-# and uses them to generate a static documentation website.
-
-# Requirements:
-# - https://github.com/sharkdp/fd - TODO: switch to `find`
-# - https://github.com/VirtusLab/scala-cli - TODO: switch to `awk`
-# - GNU sed
-# - https://gitlab.com/antora/antora
-
-# From https://stackoverflow.com/a/246128
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-# DEBUG=1 ./contrib/gendocs.sh will trace the script's execution
-if [[ -n "$DEBUG" ]]
-then
-  set -x
-fi
-
-set -euo pipefail
-trap "echo 'error: Script failed: see failed command above'" ERR
-
-# Switch to the projects root dir
-cd "$SCRIPT_DIR/.."
-
-# Idempotency
-cleanup() {
-  mv test/tests.asciidoc test/README.asciidoc
-  mv index.asciidoc README.asciidoc
-  mv VIMTOKAK.asciidoc VIMTOKAK
-}
-
-trap cleanup EXIT
-
-# Antora does not like missing links
-rm -rf libexec
-
-# Canonical Antora paths
-mkdir -p modules/ROOT/{images,pages}
-
-# Better naming
-mv test/README.asciidoc test/tests.asciidoc
-
-# The main page
-mv README.asciidoc index.asciidoc
-
-# We want this in the documentation as well
-mv VIMTOKAK VIMTOKAK.asciidoc
-
-# Put necessary images to the Antora canonical directory
-cp doc/*.gif modules/ROOT/images/
-
-# Create directories structure matching the project's original structure
-fd --extension=asciidoc --exec mkdir -p 'modules/ROOT/pages/{//}'
-
-# Take all `.asciidoc` files and put them into Antora's canonical directory with the canonical `.adoc` file extension
-fd --extension=asciidoc --exec cp {} 'modules/ROOT/pages/{.}.adoc'
-
-# Fix images
-fd --extension=adoc --exec \
-  sed --in-place --regexp-extended 's%image::doc/%image::%' '{}'
-
-# Fix links using ScalaCLI script.
-# Unfortunately it was way too hard to achieve using `awk` or `sed`.
-cat > links-fixer.scala <<EOF
-//> using scala 3.3.1
-//> using options -Werror, -Wnonunit-statement, -Wunused:all, -Wvalue-discard, -Yno-experimental, -Ysafe-init, -deprecation, -feature, -new-syntax, -unchecked,
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-import scala.jdk.CollectionConverters.*
-import scala.util.chaining.*
-
-final case class Link(
-    path: Option[String],
-    file: Option[String],
-    fragment: Option[String],
-    title: String,
-)
-
-object LinksFixer:
-
-  def main(files: Array[String]): Unit = files
-    .foreach(f => fixFile(Paths.get(f)))
-
-  private def fixFile(path: Path): Unit =
-    val file = Files.readAllLines(path).asScala.mkString("\n")
-    val _ = Files
-      .writeString(path, "<<[^>]+>>".r.replaceAllIn(file, m => fix(m.matched)))
-
-  private def fix(string: String): String =
-    if string.contains(",") then render(parse(string)) else string
-
-  private def parse(string: String): Link =
-    val trimmed = string.dropWhile(_ == '<').reverse.dropWhile(_ == '>').reverse
-    val Array(prefix, title) = trimmed.split(',')
-    val fragment = prefix
-      .dropWhile(_ != '#')
-      .trim
-      .pipe(s => Option.when(s.nonEmpty)(s))
-    val fragmentless = prefix.takeWhile(_ != '#')
-    val (path, file) =
-      if fragmentless.contains("/") then
-        fragmentless.split('/').pipe(s => (Some(s.init.mkString("/")), s.last))
-      else (None, fragmentless)
-    Link(path, Some(file), fragment, title.trim)
-
-  private def render(link: Link): String = link match
-    case Link(Some(path), Some(file), Some("#"), title) =>
-      s"xref:\$path/\$file.adoc[\$title]"
-    case Link(Some(path), Some(file), Some(fragment), title) =>
-      s"xref:\$path/\$file.adoc\$fragment[\$title]"
-    case Link(None, Some(file), Some("#"), title) =>
-      s"xref:./\$file.adoc[\$title]"
-    case Link(None, Some(file), Some(fragment), title) =>
-      s"xref:./\$file.adoc\$fragment[\$title]"
-    case Link(None, Some(file), None, title) => s"<<\$file,\$title>>"
-    case _ => throw new RuntimeException(s"Failed to render link: \$link")
-EOF
-
-fd --extension=adoc --absolute-path --exec-batch \
-  scala-cli run ./links-fixer.scala -- '{}'
-
-
-# Crafted manually from all the files at the time of writing the script.
-# TODO: Generate automatically.
-cat > modules/ROOT/nav.adoc <<EOF
-* xref:index.adoc[Getting Started]
-* xref:doc/pages/commands.adoc[Commands]
-* xref:doc/pages/expansions.adoc[Expansions]
-* xref:doc/pages/execeval.adoc[Exec Eval]
-* xref:doc/pages/scopes.adoc[Scopes]
-* xref:doc/pages/faces.adoc[Faces]
-* xref:doc/pages/buffers.adoc[Buffers]
-* xref:doc/pages/registers.adoc[Registers]
-* xref:doc/pages/mapping.adoc[Mapping]
-* xref:doc/pages/hooks.adoc[Hooks]
-* xref:doc/pages/command-parsing.adoc[Command Parsing]
-* xref:doc/pages/keys.adoc[Keys]
-* xref:doc/pages/regex.adoc[Regex]
-* xref:doc/pages/options.adoc[Options]
-* xref:doc/pages/highlighters.adoc[Highlighters]
-* xref:doc/pages/modes.adoc[Modes]
-* xref:doc/pages/keymap.adoc[KEYMAP]
-* xref:doc/pages/faq.adoc[FAQ]
-* xref:doc/pages/changelog.adoc[Changelog]
-* xref:doc/design.adoc[Design]
-* xref:doc/coding-style.adoc[Coding Style]
-* xref:doc/writing_scripts.adoc[Writing Scripts]
-* xref:doc/json_ui.adoc[JSON UI]
-* xref:doc/autoedit.adoc[Autoedit]
-* xref:doc/interfacing.adoc[Interfacing]
-* xref:rc/tools/lint.adoc[Linting]
-* xref:rc/tools/autorestore.adoc[Autorestore]
-* xref:rc/tools/doc.adoc[Doc]
-* xref:test/tests.adoc[Tests]
-* xref:VIMTOKAK.adoc[Vi(m) to Kakoune]
-EOF
-
-# Antora's component description file
-cat > antora.yml <<EOF
-name: Kakoune
-nav:
-  - modules/ROOT/nav.adoc
-title: Kakoune
-version: latest
-EOF
-
-# Antora playbook file
-cat > antora-playbook.yml <<EOF
-asciidoc:
-  attributes:
-    attribute-missing: skip # Do not complain on missing attributes, TODO: fix and turn to a fatal warning
-    idprefix: "" # To fix links
-    idseparator: "-" # To fix links
-    reproducible: true # Better to be reproducible, in general
-    sectids: true # More convenient to turn sections to IDs
-    sectlinks: true # More convenient to have sections as links
-    sectnumlevels: 5 # Do not want to miss something
-    sectnums: true # More convenient to number the sections
-    toclevels: 5 # Do not want to miss something
-  sourcemap: true
-content:
-  sources:
-  - url: ./
-    branches: ["HEAD"]
-runtime:
-  cache_dir: ./cache # More convenient for the development
-  fetch: true # More convenient for the development
-  log:
-    failure_level: fatal
-    level: warn
-output:
-  clean: true # More convenient for the development
-  dir: ./build # Simpler to have it explicit in code
-site:
-  title: Kakoune Docs
-ui:
-  bundle:
-    url: https://gitlab.com/antora/antora-ui-default/-/jobs/artifacts/HEAD/raw/build/ui-bundle.zip?job=bundle-stable   
-EOF
-
-
-# Finally, generate the documentation, will saved to the output directory specified in the antora-playbook.yml
-antora generate antora-playbook.yml
-xdg-open ./build/Kakoune/latest/index.html
diff --git a/contrib/linksfixer.py b/contrib/linksfixer.py
new file mode 100755
index 00000000..eac1fb41
--- /dev/null
+++ b/contrib/linksfixer.py
@@ -0,0 +1,83 @@
+from dataclasses import dataclass
+from typing import Optional
+import itertools
+import re
+
+
+@dataclass
+class _Link:
+    path: Optional[str]
+    file: Optional[str]
+    fragment: Optional[str]
+    title: str
+
+    def __init__(self, path, file, fragment, title):
+        self.path = path
+        self.file = file
+        self.fragment = fragment
+        self.title = title
+
+
+def fix_file(path):
+    with open(path, "r") as file:
+        content = file.read()
+
+    def fix(m):
+        string = m.group(0)
+        result = _render(_parse(string)) if "," in string else string
+        return result
+
+    with open(path, "w") as file:
+        file.write(re.sub(r"<<[^>]+>>", fix, content))
+
+
+def _str_dropwhile(predicate, string):
+    return "".join(itertools.dropwhile(predicate, string))
+
+
+def _str_dropwhileright(predicate, string):
+    return "".join(
+        reversed(list(itertools.dropwhile(predicate, reversed(string))))
+    )
+
+
+def _str_takewhile(predicate, string):
+    return "".join(itertools.takewhile(predicate, string))
+
+
+def untag(string):
+    no_opening = _str_dropwhile(lambda c: c == "<", string)
+    no_closing = _str_dropwhileright(lambda c: c == ">", no_opening)
+    return no_closing
+
+
+def _parse(string):
+    trimmed = untag(string)
+    prefix, title = trimmed.split(",", 1)
+    fragment = _str_dropwhile(lambda c: c != "#", prefix)
+    fragment = fragment if fragment else None
+    fragmentless = _str_takewhile(lambda c: c != "#", prefix)
+    segments = fragmentless.split("/")
+    path, file = (
+        ("/".join(segments[:-1]), segments[-1])
+        if "/" in fragmentless
+        else (None, fragmentless)
+    )
+    return _Link(path, (file), fragment, title.strip())
+
+
+def _render(link):
+    res = None
+    if link.path and link.file and link.fragment == "#":
+        res = f"xref:{link.path}/{link.file}.adoc[{link.title}]"
+    elif link.path and link.file and link.fragment:
+        res = f"xref:{link.path}/{link.file}.adoc{link.fragment}[{link.title}]"
+    elif not link.path and link.file and link.fragment == "#":
+        res = f"xref:./{link.file}.adoc[{link.title}]"
+    elif not link.path and link.file and link.fragment:
+        res = f"xref:./{link.file}.adoc{link.fragment}[{link.title}]"
+    elif not link.path and link.file and not link.fragment:
+        res = f"<<{link.file},{link.title}>>"
+    else:
+        raise RuntimeError(f"Failed to render link: {link}")
+    return res