#!/usr/bin/env bash
# $Id: orgdoc-check 85 2026-11-04 12:38:18Z peter $
#
# orgdoc-check --- check the Org-mode documents of sapupra against the
# markup rules of DevelopmentStandard.org that a build does not catch.
#
# Copyright (C) 2026 Péter Felecan
# Assisted by Claude Opus 5.5 (Anthropic)
#
# This program is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
# Each rule below was met as a real defect: a markup error that Org
# exports without complaint and that shows only on reading the output.
# See orgdoc-check(1).

set -euo pipefail

program=orgdoc-check
here="$(cd "$(dirname "$0")" && pwd)"

# The documents where the author may be named: every other names him
# only in its title block (DevelopmentStandard.org,
# "Documentation deliverables and their voices").
named_in="ResearchJournal.org FutureResearch.org"
# The glossary first use is checked in the documents that explain the
# project to a reader; the working records (the research journal, the
# future research list, the session handoff) and the standard, which is
# kit shared with other projects, use the words freely. Six terms are
# ordinary words (item, location, scan, status, input, edition): their
# use in the common sense cannot be told from the project's by a
# program, and they are held by review.
glossary_in="Design.org Manual.org Index.org"
glossary_review=" gl-item gl-location gl-scan gl-status gl-input gl-edition "
# The author's name is not written here: it is the first word of
# the first line of AUTHORS.
author_name="$(sed -n '1s/ .*//p' "$here/AUTHORS" 2>/dev/null)"

quiet=0
debug=0

help() {
	cat <<EOF
Usage: $program [--quiet] [--debug] [FILE.org...]
       $program -h | --help
       $program -V | --version

Check Org-mode documents against the markup rules of the development
standard that an export does not catch. With no FILE, check every
*.org of the project directory but Constitution.org (the author's own,
read-only for the assistant) and the Org files that .tar-exclude names
(Note.org, Notes.org: the author's private notes).

Rules:
  tag      a description-list tag (the text before " :: ") must be on
           the first line of its item; split over lines, the item
           silently becomes a plain item
  slash    two code or verbatim spans joined by a bare slash (~a~/~b~)
           do not parse as two spans
  space    a code or verbatim span cannot end with a space (~\$ ~);
           the markers then leak into the output
  break    a code span (between tildes) must open and close on the
           same line: a line with an odd number of tildes, once the
           verbatim spans (between equals signs) are set aside, has
           one cut by a line break
  author   only $named_in name
           the author, outside the title block
  caption  a #+CAPTION: needs a #+NAME: of the form tab:, fig: or lst:
           with it, and a #+NAME: of that form needs a caption; a name
           is defined once in a document
  label    a reference [[tab:x]], [[fig:x]] or [[lst:x]] names a
           caption of the same document
  link     a link to another document, [[file:X.org::TARGET]], lands: X.org
           exists and holds the section, the table or listing, or the
           glossary entry; a link [[*Heading]] lands in its document
  list     in FutureResearch.org: every open item (TODO, WAIT, ...) is
           in the ordered list of the Current Status with its keyword,
           every entry of the list has its item, the numbers run from 1
           without a gap, every item starts with a Ranking line, no DONE
           or DISMISSED item stays, and the Reviewed line names the
           version of the tree
  glossary in Design.org, Manual.org and Index.org: the first use, in
           prose, of a term of the glossary of Concept.org (<<gl-NAME>>)
           is a link to its entry; a word in a code span, a heading, a
           table or a block is not a use, and six ordinary words (item,
           location, scan, status, input, edition) are held by review
  decisions in FutureResearch.org: the section Decisions holds items, or,
           when none waits, only the sentence that nothing waits
           (the decisions taken are in the research journal, not here)

Each finding is printed as FILE:LINE: RULE: text. The exit status is 0
when nothing is found, 1 when something is, 2 on a usage error.

Options:
  -q, --quiet     print only the findings, not the summary
  -d, --debug     print each file as it is checked, on standard error
  -h, --help      this text
  -V, --version   print the version and exit
EOF
}

version() {
	local v="unknown"
	[[ -r "$here/VERSION" ]] && v="$(sed -n '2p' "$here/VERSION")"
	echo "$program (sapupra) $v"
}

# The rules that look across a document or across documents: captions and
# labels, links that must land, the shape of the future research list. One
# awk reads all the documents of the project twice: the first time to know
# the headings, the names and the targets, the second to check the files
# asked for.
crossdoc() {
	local -a all=() ; local f
	for f in "$here"/*.org
	do
		[[ "$(basename "$f")" == version.org ]] && continue
		all+=("$f")
	done
	local tree_version=""
	[[ -r "$here/VERSION" ]] && tree_version="$(sed -n '2p' "$here/VERSION")"
	local asked=" "
	for f in "$@"
	do
		asked+="$(basename "$f") "
	done
	awk -v asked="$asked" -v treever="$tree_version" -v glscope=" $glossary_in " -v glskip="$glossary_review" '
		function base(path) { sub(/.*\//, "", path); return path }
		function trim(t) { sub(/^[ \t]+/, "", t); sub(/[ \t]+$/, "", t); return t }
		# The text of a headline as a link names it: no stars, keyword,
		# priority, statistics cookie or tags.
		function heading(h,   kw) {
			sub(/^\*+[ \t]+/, "", h)
			kw = h; sub(/[ \t].*/, "", kw)
			if (kw ~ /^(TODO|STARTED|WAIT|RESTARTED|DONE|DISMISSED)$/) { sub(/^[A-Z]+[ \t]*/, "", h) }
			sub(/^\[#[A-Z]\][ \t]*/, "", h)
			sub(/[ \t]+:[[:alnum:]_@#%:]+:[ \t]*$/, "", h)
			sub(/[ \t]*\[[0-9]*(\/[0-9]*|%)\][ \t]*$/, "", h)
			return trim(h)
		}
		function keyword(h,   kw) {
			sub(/^\*+[ \t]+/, "", h); kw = h; sub(/[ \t].*/, "", kw)
			return (kw ~ /^(TODO|STARTED|WAIT|RESTARTED|DONE|DISMISSED)$/) ? kw : ""
		}
		function finding(rule, text) {
			if (index(asked, " " base(FILENAME) " ") > 0) {
				printf "%s:%d: %s: %s\n", FILENAME, FNR, rule, text
			}
		}
		function closekeys() {
			if (pcap != 0 && pname == "") { finding("caption", "caption without a #+NAME: of the form tab:, fig: or lst: (caption at line " pcap ")") }
			if (pname != "" && pcap == 0 && pname ~ /^(tab|fig|lst):/) { finding("caption", "#+NAME: " pname " without a #+CAPTION: (name at line " pnameline ")") }
			if (pname != "" && pcap != 0 && pname !~ /^(tab|fig|lst):/) { finding("caption", "the name of a captioned element starts with tab:, fig: or lst:, not " pname) }
			pcap = 0; pname = ""
		}
		function spans(l) { gsub(/~[^~]*~/, "", l); gsub(/=[^= \t][^=]*=/, "", l); return l }

		BEGIN { pass = 1 }
		pass == 1 {
			f = base(FILENAME)
			if (FNR == 1) { inb = 0 }
			if (tolower($0) ~ /^[ \t]*#\+begin_/) { inb = 1; next }
			if (tolower($0) ~ /^[ \t]*#\+end_/) { inb = 0; next }
			if (inb) { next }
			if ($0 ~ /^\*+[ \t]/) { H[f SUBSEP heading($0)] = 1 }
			if (tolower($0) ~ /^[ \t]*#\+name:/) { n = $0; sub(/^[ \t]*#\+[nN][aA][mM][eE]:[ \t]*/, "", n); N[f SUBSEP trim(n)] = 1 }
			l = $0
			if (f == "Concept.org" && match(l, /<<gl-[^<>]+>>[^:]+ :: /)) {
				g = substr(l, RSTART, RLENGTH); gid = g; sub(/>>.*/, "", gid); sub(/^<</, "", gid)
				gw = g; sub(/^<<[^>]*>>/, "", gw); sub(/ :: $/, "", gw); gw = tolower(trim(gw))
				if (index(glskip, " " gid " ") == 0) { nterm++; termid[nterm] = gid; termre[nterm] = "(^|[^[:alnum:]])" gw "s?([^[:alnum:]]|$)" }
			}
			while (match(l, /<<[^<>]+>>/)) { T[f SUBSEP substr(l, RSTART + 2, RLENGTH - 4)] = 1; l = substr(l, RSTART + RLENGTH) }
			E[f] = 1
			next
		}
		pass == 2 {
			if (FNR == 1) { inb = 0; sect = ""; pcap = 0; pname = ""; seen_names = ""; fr = (base(FILENAME) == "FutureResearch.org"); if (fr) { frseen = 1 } }
			if (tolower($0) ~ /^[ \t]*#\+begin_/) { closekeys(); inb = 1; next }
			if (tolower($0) ~ /^[ \t]*#\+end_/) { inb = 0; next }
			if (inb) { next }
			if (tolower($0) ~ /^[ \t]*#\+caption:/) { if (pcap == 0) { pcap = FNR } ; next }
			if (tolower($0) ~ /^[ \t]*#\+name:/) {
				n = $0; sub(/^[ \t]*#\+[nN][aA][mM][eE]:[ \t]*/, "", n); n = trim(n)
				pname = n; pnameline = FNR
				if (n ~ /^(tab|fig|lst):/) {
					if (index(seen_names, " " n " ") > 0) { finding("caption", "name defined twice: " n) }
					seen_names = seen_names " " n " "
				}
				next
			}
			if (tolower($0) ~ /^[ \t]*#\+(attr_[a-z]+|label|results):/) { next }
			closekeys()
			if ($0 ~ /^[ \t]*#/) { next }

			# links
			line = spans($0)
			while (match(line, /\[\[[^]]*\]/)) {
				tgt = substr(line, RSTART + 2, RLENGTH - 3)
				line = substr(line, RSTART + RLENGTH)
				if (tgt ~ /^(tab|fig|lst):/) {
					if (!((base(FILENAME) SUBSEP tgt) in N)) { finding("label", "reference to " tgt ", which no #+NAME: of this document defines") }
				} else if (tgt ~ /^\*/) {
					h = trim(substr(tgt, 2))
					if (!((base(FILENAME) SUBSEP h) in H)) { finding("link", "no section \"" h "\" in this document") }
				} else if (tgt ~ /^file:[^:]*\.org(::|$)/) {
					doc = tgt; sub(/^file:/, "", doc); part = doc; sub(/::.*/, "", doc); sub(/^[^:]*::/, "", part)
					if (tgt !~ /::/) { part = "" }
					doc = base(doc)
					if (!(doc in E)) { finding("link", "no document " doc) }
					else if (part ~ /^\*/) {
						h = trim(substr(part, 2))
						if (!((doc SUBSEP h) in H)) { finding("link", "no section \"" h "\" in " doc) }
					} else if (part ~ /^(tab|fig|lst):/) {
						if (!((doc SUBSEP part) in N)) { finding("link", "no " part " in " doc) }
					} else if (part ~ /^gl-/) {
						if (!((doc SUBSEP part) in T)) { finding("link", "no glossary entry " part " in " doc) }
					}
				}
			}

			# the first use of a glossary term, in prose, is a link
			if (index(glscope, " " base(FILENAME) " ") > 0 && $0 !~ /^\*/ && $0 !~ /^[ \t]*\|/) {
				for (k = 1; k <= nterm; k++) {
					if (glseen[base(FILENAME) SUBSEP termid[k]]) { continue }
					prose = spans($0)
					if (index(prose, "Concept.org::" termid[k] "]") > 0) { glseen[base(FILENAME) SUBSEP termid[k]] = 1; continue }
					gsub(/\[\[[^]]*\]\[/, "[", prose); gsub(/\[\[[^]]*\]\]/, "", prose)
					if (match(tolower(prose), termre[k])) {
						glseen[base(FILENAME) SUBSEP termid[k]] = 1
						w = substr(tolower(prose), RSTART, RLENGTH); gsub(/^[^[:alnum:]]+|[^[:alnum:]]+$/, "", w)
						finding("glossary", "the first use of \"" w "\" is not a link to Concept.org::" termid[k])
					}
				}
			}

			# the future research list
			if (fr) {
				if ($0 ~ /^\* /) { sect = heading($0); secthead = FNR; if (sect == "Decisions") { dec_line = FNR; dec_items = 0; dec_text = "" } else if (dec_line) { decide() } }
				if ($0 ~ /^\*\* / && sect == "Decisions") { dec_items++ }
				if (sect == "Decisions" && $0 !~ /^\*/) { dec_text = dec_text "\n" $0 }
				if ($0 ~ /^\*\* /) {
					kw = keyword($0)
					if (kw != "") {
						t = heading($0)
						if (kw == "DONE" || kw == "DISMISSED") { finding("list", "a " kw " item stays in the file: " t) }
						items[t] = kw; itemline[t] = FNR; lastitem = t; waitrank = 1
						next
					}
				}
				if (waitrank == 1 && $0 !~ /^[ \t]*$/) {
					if ($0 !~ /^[ \t]*Ranking:/) { finding("list", "the item " lastitem " does not start with a Ranking line") }
					waitrank = 0
				}
				if ($0 ~ /^[ \t]+[0-9]+\.[ \t]+\[\[\*/ && sect ~ /^Current Status/) {
					e = $0; num = e; sub(/^[ \t]+/, "", num); sub(/\..*/, "", num)
					title = e; sub(/^[^[]*\[\[\*/, "", title); sub(/\]\[.*/, "", title)
					kwl = e; sub(/.*\(/, "", kwl); sub(/\).*/, "", kwl)
					count++
					if (num + 0 != count) { finding("list", "the numbers of the list run " count - 1 " then " num) ; count = num + 0 }
					listed[title] = kwl; listline[title] = FNR
				}
				if ($0 ~ /Reviewed [0-9-]+ at version/) {
					v = $0; sub(/.*at version[ \t]+/, "", v); sub(/[ \t].*/, "", v)
					if (treever != "" && v != treever) { finding("list", "reviewed at version " v ", the tree is at " treever) }
					reviewed = 1
				}
			}
		}
		function decide() {
			if (dec_items == 0) {
				if (dec_text !~ /Nothing waits/) { printf "%s:%d: decisions: no decision waits and the section does not say so (\"Nothing waits for the author\x27s decision.\")\n", FILENAME, dec_line }
				else if (dec_text ~ /[0-9]+\.[0-9]+\.[0-9]+/ || dec_text ~ /\[\[/) { printf "%s:%d: decisions: a section with no decision keeps no history (versions, links): the decisions taken are in the research journal\n", FILENAME, dec_line }
			}
			dec_line = 0
		}
		END {
			if (pass == 2 && frseen) {
				if (dec_line) { decide() }
				for (t in items) {
					if (!(t in listed)) { printf "FutureResearch.org:%d: list: the item %s is not in the ordered list\n", itemline[t], t }
					else if (listed[t] != items[t]) { printf "FutureResearch.org:%d: list: the item %s is %s, the list says %s\n", listline[t], t, items[t], listed[t] }
				}
				for (t in listed) {
					if (!(t in items)) { printf "FutureResearch.org:%d: list: the list has %s, which is no item\n", listline[t], t }
				}
				if (!reviewed) { printf "FutureResearch.org:1: list: no Reviewed line\n" }
			}
		}
	' pass=1 "${all[@]}" pass=2 "$@"
}

files=()
while [[ $# -gt 0 ]]
do
	case "$1" in
		-q|--quiet) quiet=1; shift ;;
		-d|--debug) debug=1; shift ;;
		-h|--help) help; exit 0 ;;
		-V|--version) version; exit 0 ;;
		--) shift; files+=("$@"); break ;;
		-*) echo "$program: unknown option $1 (see $program --help)" >&2; exit 2 ;;
		*) files+=("$1"); shift ;;
	esac
done

if [[ ${#files[@]} -eq 0 ]]
then
	cd "$here"
	# The files the author keeps for himself (.tar-exclude, "Author
	# exclusions") are not the project's documents and are not checked.
	excluded=" "
	if [[ -r .tar-exclude ]]
	then
		while read -r pattern
		do
			case "$pattern" in
				''|'#'*|*[\*?[]*) ;;
				*.org) excluded+="$pattern " ;;
			esac
		done < .tar-exclude
	fi
	for f in *.org
	do
		[[ "$f" == Constitution.org || "$f" == version.org ]] && continue
		[[ "$excluded" == *" $f "* ]] && continue
		files+=("$f")
	done
fi

count=0
for f in "${files[@]}"
do
	if [[ ! -r "$f" ]]
	then
		echo "$program: cannot read $f" >&2
		exit 2
	fi
	[[ $debug -eq 1 ]] && echo "$program: checking $f" >&2
	reader=1
	for r in $named_in
	do
		[[ "$(basename "$f")" == "$r" ]] && reader=0
	done
	[[ -z "$author_name" ]] && reader=0
	findings="$(awk -v reader="$reader" -v author="$author_name" '
		# Source blocks and comment blocks are not prose: skip them.
		tolower($0) ~ /^[ \t]*#\+begin_(src|example|comment)/ { inblock = 1; next }
		tolower($0) ~ /^[ \t]*#\+end_(src|example|comment)/ { inblock = 0; next }
		inblock { next }
		/^[ \t]*#/ { next }

		# tag: an item opens with "- ", "+ " or "N. "; its tag must end on that line.
		/^[ \t]*([-+]|[0-9]+[.)])[ \t]/ {
			initem = 1; start = FNR; first = $0; seen = ($0 ~ / ::( |$)/)
		}
		!/^[ \t]*([-+]|[0-9]+[.)])[ \t]/ && initem {
			if ($0 ~ /^[ \t]*$/ || $0 ~ /^\*/) { initem = 0 }
			else if (!seen && $0 ~ / ::( |$)/) {
				sub(/^[ \t]+/, "", first)
				printf "%s:%d: tag: description tag split over lines: %s\n", FILENAME, start, first
				initem = 0
			}
		}

		# slash: ~a~/~b~, =a=/=b=, and mixed.
		/[~=]\/[~=]/ {
			line = $0; sub(/^[ \t]+/, "", line)
			printf "%s:%d: slash: spans joined by a bare slash: %s\n", FILENAME, FNR, line
		}

		# space: a span whose closing marker follows a space, as in ~$ ~ or =a =.
		{
			line = $0
			while (match(line, /(^|[ \t(])[~=][^ \t~=][^~=]*[ \t][~=]([ \t.,;:!?)]|$)/)) {
				printf "%s:%d: space: span ends with a space: %s\n", FILENAME, FNR, substr(line, RSTART, RLENGTH)
				line = substr(line, RSTART + RLENGTH)
			}
		}

		# break: a code span cut by a line break leaves an odd number of
		# tildes on the line, once the verbatim spans are set aside.
		{
			line = $0
			gsub(/=[^= \t][^=]*=/, "", line)
			if (gsub(/~/, "~", line) % 2 == 1) {
				line = $0; sub(/^[ \t]+/, "", line)
				printf "%s:%d: break: code span cut by a line break: %s\n", FILENAME, FNR, line
			}
		}

		# author: no author name outside the documents allowed to name him.
		reader == 1 && index($0, author) > 0 {
			line = $0; sub(/^[ \t]+/, "", line)
			printf "%s:%d: author: author named outside the journal and the future research list: %s\n", FILENAME, FNR, line
		}
	' "$f")"
	if [[ -n "$findings" ]]
	then
		echo "$findings"
		count=$((count + $(printf '%s\n' "$findings" | wc -l)))
	fi
done

cross="$(crossdoc "${files[@]}")"
if [[ -n "$cross" ]]
then
	echo "$cross"
	count=$((count + $(printf '%s\n' "$cross" | wc -l)))
fi

if [[ $count -eq 0 ]]
then
	[[ $quiet -eq 0 ]] && echo "$program: ${#files[@]} documents checked, nothing found"
	exit 0
fi
[[ $quiet -eq 0 ]] && echo "$program: $count findings in ${#files[@]} documents (see $program --help for the rules)"
exit 1
