.\"
.\" Copyright (c) 2011, 2014, 2015 Ingo Schwarze <schwarze@openbsd.org>
.\"
.\" Permission to use, copy, modify, and distribute this presentation for any
.\" purpose with or without fee is hereby granted, provided that the above
.\" copyright notice and this permission notice appear in all copies.
.\"
.\" THE PRESENTATION IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
.\" WARRANTIES WITH REGARD TO THIS PRESENTATION INCLUDING ALL IMPLIED
.\" WARRANTIES OF MERCHANTABILITY AND FITNESS.  IN NO EVENT SHALL THE
.\" AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
.\" DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA
.\" OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
.\" TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
.\" PERFORMANCE OF THIS PRESENTATION.
.\"
.\" --------------------------------------------------------------------
.\"
.\" These slides use the mm and gpresent groff macros.
.\" For example, on OpenBSD, install these ports:
.\" groff, gpresent, ghostscript.
.\" groff -mm -mpresent eurobsdcon2015-mandoc.roff > talk.pps
.\" presentps -l talk.pps > talk.ps
.\" ps2pdf talk.ps
.\"
.\" --------------------------------------------------------------------
.\"
.\" --- global mm configuration settings -------------------------------
.nr Pi 3
.\" --- global gpresent configuration settings -------------------------
.DEFCOLOR Kea1 0 0.8 0.48
.DEFCOLOR Kea2 0 0.5 0.3
.TITLECOLOR Kea1
.SUBTITLEFORMAT C
.SUBTITLECOLOR Kea2
.FOOTERSIZE 2
.\" We don't want a header line for the title page,
.\" so we have to start it before setting up headers.
.TITLE "mandoc"
.\" === gpresent header setup ==========================================
.\" --- define gpresent extension registers ----------------------------
.nr gpe_page_tot 1
.nr gpe_page_sec 0
.af gpe_page_sec I
.nr gpe_time_tsec 14*60+2*60
.nr gpe_time_hour 14
.nr gpe_time_min 02
.af gpe_time_min 02
.nr gpe_time_sec 0
.af gpe_time_sec 02
.
.\" --- macro to start a new section -----------------------------------
.de GPE_SECTION
.ds gpe_title_sec \\$1
.nr gpe_page_sec 0
..
.\" --- macro to prepare a new page ------------------------------------
.de GPE_NEXT
.ds gpe_next \\$1
.SK
..
.\" --- gpresent page header callback ----------------------------------
.de HEADER
.nr gpe_page_tot +1
.nr gpe_page_sec +1
.sp 0.5v
.ds gpe_middle page \\n[gpe_page_tot]: \\*[gpe_title_sec] \\n[gpe_page_sec]
.tl 'Ingo Schwarze: 6 years of mandoc'\
\\*[gpe_middle]'\
Stockholm, October 4, 2015'
.sp -0.5v
.\" horizontal line below the page header
\l'\\n(.lu'\h'-\\n(.lu'
.br
..
.\" --- initialize the first section before completing the title page --
.GPE_SECTION INTRO
.\" === define some gpresent extension macros ==========================
.\" --- two-column mode (for images) -----------------------------------
.\" 1st arg: width of first column
.\" 2nd arg: move second column up by this amout (default 0.5v)
.\" switch column with normal .MULN, end with normal .MULE
.de GPE_MULB
.nr gpe_colwr \\n(.l-\\$1-1n
.ie \\n[.$]>1 .ds gpe_vsp \\$2
.el .ds gpe_vsp 0.5v
.sp -\\*[gpe_vsp]
.MULB \\$1 1n \\n[gpe_colwr]u
.sp \\*[gpe_vsp]
..
.\" --- emphasis -------------------------------------------------------
.\" arg: text
.de GPE_EM
.COLOR red
\\$1
.COLOR P
..
.\" --- small text -----------------------------------------------------
.\" arg: text
.de GPE_SM
.S -4
.ce
\\$1
.S P
..
.\" --- small text with one emphasised word ----------------------------
.\" 1st arg: text before emphasis
.\" 2nd arg: emphasised text
.\" 3rd arg: text after emphasis
.de GPE_SMEM
.GPE_SM "\\$1 \m[red]\\$2\m[] \\$3"
..
.\" --- title page -----------------------------------------------------
.\" The main title line has already been printed.
.sp -1v
.SUBTITLE "from scratch to the standard BSD documentation toolkit in 6 years"
.SUBTITLE "EuroBSDCon, Stockholm, October 4, 2015"
.SUBTITLE "Ingo Schwarze <schwarze@openbsd.org>"
.ce
.GPE_SM "using some material from:"
.sp
.MULB 6.3c 1n 6.3c 1n 3.2c 1n 5.3c
.ce 4
Training a foal to replace
.br
a venerable workhorse:
.br
mandoc in OpenBSD
.br
BSDCan 2011
.MULN
.ce 5
Enhancing the modern
.br
toolbox for the classic
.br
documentation formats:
.br
new trends in mandoc
.br
BSDCan 2014
.MULN
.ce 5
Let's make
.br
manuals
.br
more useful!
.br
EuroBSD-
.br
Con 2014
.MULN
.ce 4
mandoc: becoming
.br
the main BSD
.br
manual toolbox
.br
BSDCan 2015
.MULE
.MULB 6.3c 1n 6.3c 1n 3.2c 1n 5.3c
.PSPIC Images/TomkoFoal.eps
.MULN
.PSPIC Images/BrentBarrettKea.eps
.MULN
.PSPIC Images/BSDSofiaForWeb.eps 4c
.MULN
.PSPIC Images/CynthiaLivingstonBe2013.eps
.MULE
.MULB 6.3c 1n 6.3c 1n 3.2c 1n 5.3c
.GPE_SM "Csik\('o \(em Foal \(co 2010"
.GPE_SM "Adam Tomk\('o @flickr (CC)"
.MULN
.GPE_SM "Kea juvenile \(co 2007"
.GPE_SM "Brent Barrett @flickr (CC)"
.MULN
.GPE_SM "\f[CYR]Sofi\(ya\fP \(co 2014"
.GPE_SM "\h'-0.5c'\f[CYR]Alica Dimitrova\fP"
.MULN
.GPE_SM "\(lqBedifferent\(rq \(co 2013"
.GPE_SM "Cynthia Livingston"
.MULE
.\" === gpresent footer setup ==========================================
.\" We dont want a footer line for the title page,
.\" so we have to set it up after completing the title page.
.SK
.\" --- macros to start a new page -------------------------------------
.\" arg: time for this page in seconds
.de GPE_TIME
.nr gpe_time_tsec +\\$1
.nr gpe_time_hour \\n[gpe_time_tsec]/3600
.nr gpe_time_min \\n[gpe_time_tsec]%3600/60
.nr gpe_time_sec \\n[gpe_time_tsec]%60
.ds gpe_source \\$2
..
.\" --- gpresent page footer callback ----------------------------------
.de FOOTER
.ps 18
.vs 20
.sp -2v
\l'\\n(.lu'\h'-\\n(.lu'
.br
.tl '\s-6\\n[gpe_time_hour]:\\n[gpe_time_min]:\s-2\\n[gpe_time_sec]\
 \\*[gpe_source]\s+8''\\m[Kea2]\\*[gpe_next]\ \ \(->\\m[]'
.ps
.vs
..
.\" The INTRO section was already started in header.roff.
.TITLE "Contents"
.sp -1v
.AL
.LI
Intro: Documentation \(em why and how (EuroBSDCon/BSDCan 2014)
.LI
Using mandoc:
.br
Searching \(em unified interface \(em web display (BSDCan 2014/15)
.br
News: equations \(em unicode (BSDCan 2015)
.br
Maintaining documentation: warnings \(em help \(em portable software (all)
.LI
The groff \(-> mandoc replacement project (BSDCan 2011)
.LI
Software isn't perfect.
.br
Bugs, security issues, performance (BSDCan 2015/14)
.LI
Conclusion \(em status \(em future \(em thanks (BSDCan 2015)
.LE
.sp 0.5v
.ce
.GPE_EM "http://mdocml.bsd.lv/press.html has all the slides of these talks"
.sp -1.5v
.PSPIC Images/LezumbalaberenjenaBlackLake.eps
.GPE_SM "Black Lake near King Mountain, Gatineau Park, Quebec, Canada\
 \(co 2012 Lezumbalaberenjena@flickr (CC)"
.GPE_TIME 150 "NYC*BUG 2015"
.GPE_NEXT "Why document software?"
.GPE_SMEM "Let's make manuals more" useful! ""
.sp -0.5v
.TITLE "Requirements for good documentation"
.sp -0.5v
.GPE_MULB 14c
.BL
.LI
correct
.LI
complete
.LI
concise
.LI
easy to find and access,
.br
all in one local place
.LI
not just plain text:
function of words
.br
must be marked up for display and search
.LE
.MULN
.PSPIC Images/StGeorgeRotunda.eps
.GPE_SM "\f[CYR]Rotonda Sveti Georgi, Sofi\(ya\fP"
.GPE_SM "\(co 2006 \f[CYR]Preslav\fP @wikimedia (PD)"
.MULE
.BL
.LI
easy to read:
in particular, uniform display markup and style
.LI
easy to write:
in particular, one simple, standard input language
.LE
.sp
The formatted documentation must seem simple to end users.
.P
The language to write documentation must seem simple to programmers.
.sp
.ce 3
Remember:
.GPE_EM "Without documentation, code is unusable,"
.GPE_EM "and bad documentation is about as bad as bad code."
.GPE_TIME 100 "EuroBSDCon 2014 p. 2"
.GPE_NEXT "How does the user experience manuals?"
.GPE_SMEM "Let's make" manuals "more useful - the user's perspective."
.sp -0.5v
.TITLE "Let's read a manual!"
.sp -0.5v
.GPE_MULB 13c 1v
One comprehensive tool:
.br
\fBman(1)\fP, the manual viewer
.AL
.LI
Find one or more manuals
.br
in the database or file system.
.LI
Transparently call a formatter on them.
.LI
Display the formatted text,
.br
typically in a pager.
.LE
.MULN
.PSPIC Images/StelianKasabovMaljovitsa.eps
.GPE_SM "\f[CYR]Malyovica 2729m, Rila\fP, Bulgaria"
.GPE_SM "\(co 2007 \f[CYR]Steli\(yan Kasabov\fP @flickr (PD)"
.MULE
.sp -1v
Progress during the last year:
.BL
.LI
The mandoc toolbox now provides one single tool for all this.
.LI
Unified, simple
.GPE_EM "user interface"
available since August 26, 2014.
.LI
The traditional way required multiple programs:
apropos for searching, man for steering, nroff or mandoc for formatting, ...
.LI
Semantic searching
.GPE_EM "of pages"
with apropos(1) in production since April 14, 2014.
.LI
Semantic searching
.GPE_EM "within a page"
in ctags(1) style since July 17, 2015.
.LI
New man.cgi(8) is
.GPE_EM online
on www.openbsd.org since July 12, 2014.
.LE
.GPE_TIME 80 "EuroBSDCon 2014 p. 3 (updated)"
.GPE_NEXT "What do authors use?"
.GPE_SMEM Let's make "manuals more useful - the author's perspective."
.sp -0.5v
.TITLE "Let's write manuals!"
.GPE_MULB 12c 1v
So we need a markup language that is:
.BL
.LI
easy to write
.LI
easy to diff(1)
.LI
easy to read and change
.LI
easily supports semantic markup
.LI
readily produces various output formats
.LI
easily supports portability
.LE
.MULN
.PSPIC Images/KirilRusevKutelo.eps 8c
.GPE_SM "\f[CYR]Kutelo 2908m i Vihren 2914m, Pirin\fP"
.GPE_SM "\(co 2010 \f[CYR]Kiril Rusev\fP @flickr (CC)"
.MULE
.sp
One simple, versatile language: \fBmdoc(7)\fP \(em
with a long tradition:
.BL
.LI
Based on the roff(7) language (Jerome Saltzer, 1964).
.LI
Successor of the man(7) language (AT&T v7 UNIX, 1979).
.LI
Designed for 4.4BSD at Berkeley (Cynthia Livingston, 1989/90).
.LI
Implemented by mandoc(1) (2008) and groff(1) (1989).
.LI
.GPE_EM "Used by default in OpenBSD, NetBSD, FreeBSD, DragonFly and illumos."
.LE
.GPE_TIME 90 "EuroBSDCon 2014 p. 4 (slightly extended)"
.GPE_NEXT "Was there any progress during the last 35 years?"
.GPE_SMEM "Enhancing the" modern\
 "toolbox for the \m[red]classic\m[] documentation formats:
.TITLE "Origin of semantic markup in manuals"
.GPE_MULB 18c 2v
.BL
.LI
The mdoc(7) semantic markup macro language.
.LI
First in 4.3BSD-Reno by Cynthia Livingston, USENIX, 1990.
.LI
Formatted with Brian Kernighan's device independent troff,
.LI
written in K&R C, running on BSD UNIX on DEC VAX.
.LE
.SUBTITLE "Advantages of the mdoc(7) language"
.MULN
.PSPIC Images/BSDBeastie.eps
.GPE_SM "4.2BSD Beastie"
.MULE
.BL
.LI
Considerable expressive power for semantic markup \(em
.br
while man(7) is a presentation level language only.
.LI
Works in practice as a standalone language \(em
.br
while man(7) regularly requires resorting to low-level roff features.
.LI
Consequently, more uniform appearance, easier to read and write than man(7).
.LI
Portability is no longer an issue:
for legacy systems still not having mdoc(7),
mandoc(1) can be used to convert to man(7).
.LI
.GPE_EM "Facilitates semantic searching \(em see this talk."
.LE
.GPE_TIME 90 "BSDCan 2014 p. 7 (slightly modified)"
.GPE_NEXT "Classic documentation formats (summary)"
.TITLE "Classic documentation formats (summary)"
.BL
.LI
The roff(7) input syntax,
.LI
the mdoc(7) semantic markup,
.LI
and the man(1) presentation format
.LE
.sp 0.5v
.ce
.GPE_EM "have proven timeless by their simplicity and efficiency:"
.sp -0.5v
.BL
.LI
Nobody has come up with a better basic concept yet,
.LI
even though many have tried,
.LI
and regarding the formats, there is indeed little one could wish.
.LE
.PSPIC Images/AndrewRockWren.eps 10c
.ce
.GPE_SM "Rock Wren / Hurupounamu (Xenicus gilviventris) New Zealand\
 \(co 2006 57Andrew@flickr (CC)"
.GPE_TIME 30 "BSDCan 2014 p. 9"
.GPE_NEXT "So we need modern tools for all this!"
.GPE_SMEM Enhancing "the modern toolbox"\
 "for the classic documentation formats:"
.TITLE "Advantages of mandoc(1)"
.BL
.LI
Functional \(em all in one binary:
.BL
.LI
searching by filename, page name, word, substring, regular expression
.LI
searching by semantic keys in and across pages
.LI
mdoc(7), man(7), tbl(7), eqn(7), and some roff(7) input
.LI
ASCII, UTF-8, HTML 5, PostScript, PDF output
.LI
mdoc(7) to man(7) conversion
.LI
includes mandoc(1), man(1), apropos(1), whatis(1), makewhatis(8)
.LE
.LI
Free \(em ISC/BSD-licensed, no GPL code.
.LI
Lightweight \(em ANSI C, POSIX, no C++ code.
.LI
Portable \(em includes compat_*.c files
for missing functions on older systems.
.LI
Small \(em source tarball (uncompressed) is 8% of groff, executable binary 50%.
.LI
Fast \(em for mdoc(7), typically 5 times faster than groff, typically
about a hundred times faster than an AsciiDoc/DocBook toolchain.
.LE
.sp
.ce
.GPE_EM "Basic functionality already presented during BSDCan 2011."
.GPE_TIME 90 "BSDCan 2014 p. 10 (updated)"
.GPE_SECTION "USING MANDOC"
.GPE_NEXT "How does searching work?"
.TITLE "Searching for manual pages"
.sp -1.5v
.SUBTITLE "Traditional functionality is preserved"
.sp -0.5v
.BVL 1c
.LI "apropos \fIkeywords\fP ..."
Search case-insensitively for substrings in names and descriptions.
.LI "man \-k \fIarguments\fP"
As before, an alias for: apropos \fIarguments\fP
.br
But now also supports new-style arguments, see below.
.LI "apropos [\fB\-C \fIfile\fR] [\fB\-M \fIpath\fR] [\fB\-m \fIpath\fR] \
[\fB\-S \fIarch\fR] [\fB\-s \fIsection\fR] \fIkeywords\fR ..."
Traditional options are all supported.
.LE
.GPE_MULB 12c
.BVL 1c
.LI "whatis \fIkeywords\fP ..."
Search case-insensitively for complete
.br
words in page names only.
.LI "makewhatis"
Rebuild all configured databases.
.LI "makewhatis \-d \fIdirectory files ...\fP
Update entries for the given \fIfiles\fP
.br
in one database.
.LE
.P
.ce
.GPE_EM "backward compatible"
.sp -2v
.MULN
.PSPIC Images/GlenFergusKiwi.eps
.ce 2
.GPE_SM "Southern Kiwi / Tokoeka (Apteryx australis)"
.GPE_SM "\(co 2008 Glen Fergus @wikimedia (CC)"
.sp -2v
.MULE
.GPE_TIME 60 "BSDCan 2014 p. 12"
.GPE_NEXT "New functionality"
.TITLE "Markup-sensitive search"
.sp -1v
.VERBON 0 12
 $ apropos Ev=USER
Mail, mail, mailx(1) \(en send and receive mail
csh(1) \(en a shell (command interpreter) with Clike syntax
login(1) \(en log into the computer
logname(1) \(en display user's login name
slogin, ssh(1) \(en OpenSSH SSH client (remote login program)
su(1) \(en substitute user identity
[...]
.VERBOFF
.sp 0.2v
Macro keys that can be searched for (examples, ordered by frequency):
.GPE_MULB 5i
.S -4
.VL 2c 0c 1
.LI Nm
manual page names
.LI Nd
manual page descriptions
.LI sec
manual section numbers
.LI arch
machine architectures
.LI Xr
cross references
.LI Ar
command argument names
.LI Fa
function argument types and names
.LI Dv
preprocessor constants
.LI Pa
file system paths
.LI Cd
kernel configuration directives
.LI Va
variable names
.LI Ft
function return types
.LI Er
error constants
.LI Ev
environment variables
.LI In
include file names
.LI St
references to standards documents
.LI An
author names
.LI ...
and so on ...
.LE
.S +4
.MULN
.PSPIC Images/JJHarrisonSilvereye.eps (u;\n(.l-\n(.i)u
.ce 2
.GPE_SM "Silvereye / Tauhou (Zosterops lateralis)"
.GPE_SM "\(co 2008 J. J. Harrison @wikimedia (CC)"
.MULE
.GPE_TIME 90 "BSDCan 2014 p. 13"
.GPE_NEXT "Intermediate new search features"
.TITLE "Markup-sensitive search features"
Search keys can be OR'ed:
.VERBON 6 12
 $ apropos \m[red]Fa,Ft,Va,Vt=\m[]timespec
EV_SET, kevent, kqueue(2) \(en kernel event notification mechanism
clock_getres, clock_gettime, clock_settime(2) \(en get/set/calibrate date and time
futimens, futimes, utimensat, utimes(2) \(en set file access and modification times
nanosleep(2) \(en high resolution sleep
parse_time(3) \(en parse and unparse time intervals
poll, ppoll(2) \(en synchronous I/O multiplexing
pselect, \m[red]select(2)\m[], FD_CLR, FD_ISSET, FD_SET, \
\m[red]FD_ZERO(3)\m[]\ \(en synchronous I/O multiplexing
sem_timedwait, sem_trywait, sem_wait(3) \(en decrement (lock) a semaphore
tstohz, tvtohz(9) \(en translate time period to timeout delay
[...]
.VERBOFF
Searching across all keys is possible:
.VERBON 6 12
 $ apropos \m[red]any=\m[]ulimit
ksh, rksh(1) \(en public domain Korn shell
sh(1) \(en public domain Bourne shell
getrlimit, setrlimit(2) \(en control maximum system resource consumption
.VERBOFF
Regular expressions are supported (\(oq~\(cq instead of \(oq=\(cq)
since October 19, 2013:
.VERBON 6 12
 $ apropos "Nm~^[gs]et.*gid"
endgrent, getgrent, getgrgid, getgrgid_r, getgrnam, getgrnam_r, setgrent, setgrfile, setgroupent(3) \(en group database operations
getegid, getgid(2) \(en get group process identification
getpgid, getpgrp(2) \(en get process group
getresgid, getresuid, setresgid, setresuid(2) \(en get or set real, effective and saved user or group ID
setegid, seteuid, setgid, setuid(2) \(en set user and group ID
setpgid, setpgrp(2) \(en set process group
setregid(2) \(en set real and effective group IDs
[...]
.VERBOFF
.GPE_TIME 50 "BSDCan 2014 p. 14"
.GPE_NEXT "Why was man(1) replaced?"
.TITLE "A better manual viewer"
.GPE_MULB 15c
.SUBTITLE "Current advantages"
.BL
.LI
.GPE_EM "Unified interfaces:"
.br
man can use: -W -T -O -I from mandoc(1)
.br
apropos can use: -w -h -a from man(1)
.ig
  man -Tlint crypt_checkpass     # check an installed manual
  man -Thtml lynx | lynx -stdin  # pager overkill
  man -Tps gv | gv -             # even more pager overkill
..
.LI
Allow much simpler man.conf(5) format
.LI
Database priority now overrides section priority
.LI
Use additional names from .Dt/.TH and NAME .Nm
(250 new entries in OpenBSD, compared to 10200 existing ones)
.LI
One less userland program to maintain
.br
(that had rather old code)
.LE
.MULN
.PSPIC Images/DougKerrEganville.eps
.GPE_SM "Bonnechere Museum, Eganville"
.GPE_SM "\(co 2011 Doug Kerr @flickr (CC)"
.MULE
.SUBTITLE "Possible future advantages"
.BL
.LI
Possibility to get rid of multiple ln(1) links to the same manual
.LI
Possibility to implement an interactive chooser (-i)
.LE
.GPE_TIME 100 "BSDCan 2015 p. 8 (slightly modified)"
.GPE_NEXT "Are manuals only available at the console?"
.TITLE "Web interface for manual search and display"
.GPE_MULB 11.5c
.BL
.LI
Same user interface as on the command line:
man(1) and whatis(1) mode, same query syntax
.LI
Main additional feature: hyperlinks
.LI
Additional potential due to preserved semantic markup,
not really used yet for anything except (simple) CSS formatting
.LI
Same directory structure and database format on the server
.LI
Configuration instructions in man.cgi(8)
.LE
.MULN
.PSPIC Images/CharlesjsharpRedFoxKit.eps
.ce 2
.GPE_SM "Red fox kit (Vulpes vulpes)"
.GPE_SM "\(co 2014 Charlesjsharp @wikimedia (CC)"
.MULE
.sp 0.5v
.BL
.LI
Useful mainly for providers of operating systems and large software packages.
.LI
Running your own copy of the manuals of your favourite system is a bad idea:
.LI
It will get outdated and confuse people.
.LE
.sp
.ce
.GPE_EM "Special thanks to \fIS\('ebastien Marie\fP:"
He did an extensive security audit of the
.Xr man.cgi 8
code and reported a considerable number of security-relevant bugs
that have all been fixed by now.
.GPE_TIME 30 "EuroBSDCon 2014 p. 46"
.GPE_NEXT "What is internal searching?"
.TITLE "Internal searching and deep linking"
.ce
.GPE_EM Why?
.BL
.LI
Open a large manual page, like ksh(1) or perlfunc(1).
.LI
Where is the description of a keyword like
.B set ,
.B if ,
.B print ?
.LI
Searching with "/" in a pager is almost hopeless.
.LE
.sp
.ce
.GPE_EM "Imagine just saying: Go to the description of ""set""!"
.GPE_MULB 10c
.sp
Multi-dimensional task:
.BL
.LI
link targets
.LI
link sources
.LI
link distance
.LI
jumping mechanism
.LI
output media
.LE
.MULN
.PSPIC Images/ElpocaRock.eps
.GPE_SM "Elpoca Mountain 3029m from the Rock Glacier 2100m"
.GPE_SM "Kananaskis Trail, Alberta \(co 2015 Ingo Schwarze"
.MULE
.GPE_TIME 60 NEW
.GPE_NEXT "What are the dimensions?"
.TITLE "Dimensions of internal searching and deep linking"
.VL 20n
.LI "link targets:"
\(em sections and subsections:   .Sh .Ss
.br
\(em syntax element descriptions:   .Fn .Ic .Cm .Ev .Er ...
.br
\(em anchors specified by the document author (sparingly)
.br
.I "The less explicit, the better: avoid requiring new syntax."
.LI "link sources:"
\(em explicitly specified by the document author (sparingly)
.br
\(em  implicit: mention of syntax element \(-> description
.br
\(em or just from the reader's mind (= internal searching)
.LI "link distance:"
\(em same document (local deep links)
.br
\(em different document (remote deep links)
.LI "jumping mechanism:"
\(em  search style: type in the target name
.br
\(em  hyperlink style: click the link (or type ID or #?)
.LI "output media:"
terminal, HTML, PDF, ...
.br
We don't want mouse-clicking on a server serial console.
.br
We don't want to type link numbers in a graphical browser.
.br
Yet, we do want consistent functionality across all media!
.LE
.P
.GPE_EM "It's crucial to not jump to an implementation:"
.br
The first specific proposal (by Steffen Nurpmeso) would have doubled
the number of macros and required substantial rewriting of all
manuals... \(em Yikes!
.GPE_TIME 150 NEW
.GPE_NEXT "What was done so far?"
.TITLE "Done: local searching for implicit targets"
.GPE_MULB 10.5c
.GPE_EM "State before i started:"
.BL
.LI
local explicit links (.Sx) to sections
.br
and subsections (.Sh .Ss)
.LI
only working in HTML output mode
.LI
only linking, no search functionality
.br
(except /^SECNAME<enter>)
.LE
.sp
.GPE_EM "What i did this summer:"
.P
Designed a way to work with
.br
a list of local targets.
.MULN
.PSPIC Images/MistLineham.eps 9c
.GPE_SM "Mist Mtn. (3138m) from Lineham Creek Bridge (1680m)"
.GPE_SM "Elbow-Sheep Wildland Provincial Park, Kananaskis,"
.GPE_SM "Alberta, Canada \(co 2015 Ingo Schwarze"
.MULE
So far used for implicit targets only:
.P
Definitions of syntax elements (functions, keywords, flags, variables, errors)
.br
So far used for local searching only
(no linking/nothing remote yet; both non-trivial)
.P
Having a way to handle targets
.I "at all"
is a crucial step, and logical as a first step!
.P
It's already quite useful. \(-> live-demo
.GPE_TIME 120 NEW
.GPE_NEXT "How does it work?"
.TITLE "Implementation of local jump targets"
When using a pager and at least one page is formatted
.br
(otherwise, nothing changes, just use stdout; no change for  build systems):
.AL
.LI
.B tag_init ():
set up two temporary files (output, tags) and one ohash table
.LI
during formatting:
.BL
.LI
count output lines (depends on
.B -Owidth ,
info cannot be in a database)
.LI
\fBtag_put\fP(\fIconst char *s\fP, \fIint prio\fP, \fIsize_t line\fP)
for .Cm .Er .Ev .Fl .Fn .Ic
.LE
.LI
.B tag_write ()
.LI
.B spawn_pager ():
call
.B "less -T"
.I "tag_file out_file"
.LI
.B waitpid ()
until the user terminates less
.LI
.B tag_unlink ()
.LE
.sp
.GPE_MULB 12c 3c
Implementation size:
.BL
.LI
40 new lines in existing code (formatter)
.LI
240 new lines in a one module
.I tag.h ,
.I tag.c
.LE
.sp
User interface bloat:
.BL
.LI
no new macros, no new syntax whatsoever
.LE
.MULN
.PSPIC Images/YellowBelliedMarmot.eps 8.5c
.GPE_SM "Yellow Bellied Marmot, Canyon Campground"
.GPE_SM "Kananaskis, Alberta \(co 2015 Ingo Schwarze"
.MULE
.GPE_TIME 100 NEW
.GPE_NEXT "What next?"
.TITLE "Plans for internal searching and deep linking"
.sp -0.5v
.ce
.GPE_EM "Carefully and slowly extend along the various dimensions"
.sp -0.5v
.VL 20n
.LI "link targets:"
add more implicit targets (simple + some smart heuristics)
.br
support explicit anchor definitions (straightforward)
.LI "link sources:"
.GPE_EM carefully
design explicit internal links to arbitrary targets
.LI "jumping mechanism:"
.GPE_EM carefully
design a concept for console hyperlinks
.br
\(em then use that for the existing .Sx
.br
\(em and for implicit links from mentions to descriptions
.LI "link distance:"
.GPE_EM carefully
design a concept for remote deep links
.br
\(em then use that for .Xr to specific .Sh/.Ss
.br
\(em then use that for .Xr to other, e.g. implicit targets
.LI "output media:"
design an analogous concept to :t for HTML (and PDF?)
.LE
.sp -0.5v
.GPE_MULB 14c
.PSPIC Images/CalgaryKensington.eps 12c
.MULN
.sp 3c
.S -4
The Calgary skyline,
.br
looking SE over Kensington
.br
along 7th Street NW,
.br
Alberta, Canada
.br
\(co 2015 Ingo Schwarze
.S P
.MULE
.GPE_TIME 120 NEW
.GPE_NEXT "What else is in the toolbox?"
.TITLE "Auxiliary components in the toolbox"
.BL
.LI
makewhatis(8) database generation and maintenance
.LI
mandoc(1) -Tlint syntax checker
.LI
mandoc(1) -Thtml polyglot HTML5 generator
.LI
mandoc(1) -Tpdf and -Tps formatters
.LI
mandoc(1) -Tman format converter
.LI
mandoc(1) -Ttree parse tree debugger
.LI
soelim(1) file inclusion resolver
.LI
\&...
.LE
.PSPIC Images/MacArmstrongKingEstate.eps 8c
.GPE_SM "Moorside Cottage, Mackenzie King Estate, Gatineau Park\
 \(co 2009 Mac Armstrong @flickr (CC)"
.GPE_TIME 10 "BSDCan 2015 p. 4"
.GPE_NEXT "What about equations?"
.TITLE "Equations in manual pages"
.sp -0.5v
.BVL 1c
.LI "Relevance of the eqn(7) language:"
Used less than mdoc(7), man(7), and tbl(7); but: X.org!
.LI "Problem was: parsing works well (since 2011), formatting was ugly..."
Representing mathematical formula on the terminal is hard.
.LI "HTML output:"
Rewrite to generate
.GPE_EM MathML
(Kristaps Dzonsons, Oct 10):
.br
straightforward, 1:1 translation of the syntax tree,
less than 200 lines of code
.br
quite beautiful in a graphical browser
.LI "Terminal output:"
.BL
.LI
GNU eqn: moves elements up and down to the previous or next line and
draws bars out of ASCII characters \(em results are unintelligible
.LI
mandoc output rewrite (Ingo Schwarze, Oct 12):
.br
.GPE_EM "linear textual representation:"
showing fractions like (a + b)/(c + d),
.br
matrices like ((a_11 a_12)(b_21 b_22)), and roots like sqrt(x)
.LI
certainly not pretty, but at least you can figure out what the formulas mean
.LI
merely 125 lines of very straightforward code
.LE
.LE
.P
mandoc eqn now much better than GNU eqn for both terminal and HTML
.br
PostScript/PDF is still the domain of GNU eqn:
mandoc just does the same as on the terminal.
.GPE_TIME 120 "BSDCan 2015 pp. 19-20 (shortened)"
.GPE_NEXT "What about UTF-8?"
.TITLE "Multibyte character support"
.sp -0.5v
.BVL 1c
.LI "Non-english manuals:"
rare \(em hard to maintain \(em worse than nothing when outdated...
.LI "But the tools must not hinder reading them!"
That would only aggravate the problem.
.LI "Hence, early basic UTF-8 support (Kristaps Dzonsons, May 20-26, 2011):"
Like groff, use a preconv(1) preprocessor: UTF-8 \(-> roff escape sequences
.br
and -Tutf8 and -Tlocale terminal output modes.
.LE
.SUBTITLE "Improvements in 2014"
.\" 2013 Oct 5 release 1.12.2: fix -Tutf8 and more (Ingo Schwarze) 
.GPE_MULB 9cm
.BL
.LI
integrate preconv(1):
.br
direct UTF-8 input (Oct 25)
.LI
default to -Tlocale (Dec 2)
.br
(formerly -Tascii)
.LI
.GPE_EM "No more options needed!"
.LI
released with 1.13.2 (Dec 13)
.LI
rewrite UTF-8 parser (Dec 19)
.LI
all by Ingo Schwarze
.LE
.MULN
.PSPIC Images/JoeMabelWasaga.eps 10.5c
.GPE_SM "Wasaga Beach, L. Huron \(co 2013 Joe Mabel @wikimedia (CC)"
.MULE
.GPE_TIME 110 "BSDCan 2015 p. 21"
.GPE_NEXT "Why does the design of -Tlint messages matter?"
.TITLE "Why it is critical to get errors and warnings right"
.sp -0.5v
.GPE_MULB 13c
.BVL 5n
.LI "A fatal error gets thrown:"
The manual doesn't format at all,
.br
which is very inconvenient.
.br
Can no longer happen since Jan 15, 2015.
.LI "An error message is missing or too generic:"
Users have a hard time to fix their errors. 
.LI "A warning message is missing:"
Users don't even notice
.br
their dangerous idioms. 
.LI "A few warnings too many, or too prominent:"
Users get annoyed
.br
and switch off all warnings. 
.LI "More than one or two knobs:"
Users don't remember and don't use them. 
.LE
.MULN
.PSPIC Images/JeffBurcherHorseFly.eps
.GPE_SM "Horse Fly Portrait"
.GPE_SM "\(co 2010 Jeff Burcher @flickr (CC)"
.MULE
.GPE_EM "Too few and too many can happen all at once!"
.br
It did with mandoc, and it had too many knobs - at first.
.P
There were several major message cleanups in mandoc:
July 2009, May 2010, August 2010, October 2010, January 2011,
March 2011, July 2014, January 2015, ...
.GPE_TIME 150 "BSDCan 2011 p. 21 (updated)"
.GPE_NEXT "Where can we get more information?"
.TITLE "Help on mdoc(7)"
.GPE_MULB 16c
.BL
.LI
The mdoc(7) manual is the most important resource.
.BL
.LI
which sections? \(em MANUAL STRUCTURE section
.LI
which macros? \(em MACRO OVERVIEW section
.LI
usage of a macro? \(em MACRO REFERENCE section
.LE
.LI
Extended mdoc documentation:
.br
http://mdocml.bsd.lv/mdoc/
.LI
Look at OpenBSD base system manuals for examples.
.br
This is particularly helpful to find standard wordings
.br
and customary choices of macro arguments.
.br
For example, many pages contain this:
.nr Verbin 10n
.VERBON 22
The options are as follows:
\&.Bl -tag -width Ds
\&.It Fl a
.VERBOFF
.nr Verbin 5n
.LE
.MULN
.PSPIC Images/NZDOCKakapoChicks.eps
.GPE_SM "Kakapo chicks"
.GPE_SM "(Strigops habroptilus)"
.GPE_SM "(c) 2009 NZ DOC @flickr (CC)"
.MULE
.BL
.LI
.B "mandoc -Tlint"
catches most syntax errors and provides some hints on style.
.LI
The groff_mdoc(7) manual sometimes helps to resolve ambiguities.
.LI
Kristaps has written a full tutorial: http://manpages.bsd.lv/
.LI
Ask for help on <discuss@mdocml.bsd.lv>
.br
and provide suggestions to improve the mdoc(7) manual.
.LE
.GPE_TIME 90 "EuroBSDCon 2014 p. 21 (updated)"
.GPE_NEXT "Can we use this for portable software?"
.TITLE "Manual pages for portable software"
.SUBTITLE "The problem"
.BL
.LI
Consider portable software packages like sudo(8), OpenSSH, OpenSMTPd, ...
.LI
Which markup language should be chosen for the manual pages?
.LI
Use mdoc(7) and some legacy systems lose that still don't have mdoc(7)
after it has been freely available for more than 20 years (hello, Solaris).
.LI
Use man(7) and everybody loses \(em that would be a very bad idea indeed.
.LE
.SUBTITLE "mandoc(1) to the rescue!"
.BL
.LI
.GPE_EM "Write the manual pages using mdoc(7)."
.LI
Use mandoc -Tman to convert them to man(7) format.
Fully operational since November 19, 2012.
.LI
Include both the mdoc(7) and man(7) versions into distribution tarballs.
.LI
Let \f(CW./configure\fP decide:
.LI
On systems supporting mdoc(7), install the mdoc(7) versions.
.LI
Otherwise, install the man(7) versions.
.LE
.GPE_TIME 90 "BSDCan 2014 p. 33"
.GPE_NEXT "What's the problem with ports manuals?"
.TITLE "Handling manual pages in ports"
.SUBTITLE "The problem"
.BVL 1c
.LI "The Law of Feature Creep"
If a software offers some feature,
sooner or later somebody will use it.
.LI "Porting corollary"
For every feature of the roff language (and for every groff extension),
no matter how arcane and how obviously irrelevant for manual pages,
sooner or later somebody will want to port a third-party software
abusing that feature to format its manual pages.
.LI "mandoc(1) is not a complete nroff implementation"
and who knows whether it will ever be...
.sp
.LI "Not a problem in the base system:"
Given a finite set of manuals, implement in mandoc(1) what is needed,
or patch away the worst abuse in the handful of manuals affected.
.LI "\m[red]But in ports, \(lqmandoc or nothing\(rq is not a viable\
 strategy:\m[]"
That would inevitably leave you with some seriously misformatted manuals.
.LE
.GPE_TIME 110 "BSDCan 2014 p. 30 (slightly updated)"
.GPE_NEXT "How does OpenBSD handle manual pages in ports?"
.TITLE "Manual pages in ports"
.ce
By now, 95% of ports manuals just work with mandoc.
.SUBTITLE "The OpenBSD way"
.BL
.LI
Mark those that don't work individually with USE_GROFF (about 200 remain).
.LI
Preformat these manuals at port build time with groff,
.br
.GPE_EM "package the preformatted versions"
of the manuals.
.LE
.GPE_MULB 9c
.sp 2v
Advantage:
.br
Perfect manuals for every port.
.sp 2v
Inconveniences:
.br
Needs support in the
.br
ports infrastructure
.br
(written by Marc Espie@),
.sp 0.5v
and manual maintenance
.br
of the USE_GROFF variable.
.MULN
.PSPIC Images/PaulBicaHumberBridge.eps 12c
.GPE_SM "Humber Bay Bridge, Toronto \(co 2010 Paul Bica @flickr (CC)"
.MULE
.GPE_TIME 60 "BSDCan 2015 p. 27"
.GPE_SECTION "REPLACEMENT"
.GPE_NEXT "How does a replacement project work?"
.TITLE "Block nesting"
.SUBTITLE "An example of nice mandoc design"
.MULB 9c 1n 6c 1n 6c
Example manual snippet
.br
using nested blocks:
.DS
.ft CW
.ps -4
$ man chgrp
SYNOPSIS
  chgrp [-h] [-R [-H|-L|-P]]
        group file ...
.ft P
.ps +4
.DE
.P
.GPE_EM "Traditional roff/mdoc design:"
.br
.GPE_EM "No block structure!"
.P
.B "Low level: roff requests provide"
.BL
.LI
physical formatting
.LI
registers to store data
.LI
macro expansion 
.LE
.P
.B "High level: mdoc macros"
.BL
.LI
call physical formatting
.LI
set registers to keep state
.LI
have no syntax tree
.LE
.MULN
mdoc(7) source code:
.sp
.DS
.ft CW
.ps -4
$ less `man -w chgrp`
[...]
\&.Sh SYNOPSIS
\&.Nm chgrp
\&.Op Fl h
\&.Oo
\&.Fl R
\&.Op Fl H | L | P
\&.Oc
\&.Ar group
\&.Ar
[...]
.ft P
.ps +4
.DE
.MULN
Syntax tree:
.br
(in mandoc, simplified)
.DS
.ft CW
.ps -4
$ man -Ttree chgrp
Sh (block) SYNOPSIS
  Nm (block) chgrp
    Op (block)
      Fl (elem) h
    Oo (block)
      Fl (elem) R
      Op (block)
        Fl (elem) H
        (text) |
        Fl (elem) L
        (text) |
        Fl (elem) P
    Ar (elem) group
    Ar (elem) file ...
.ft P
.ps +4
.DE
.MULE
.GPE_TIME 120 "BSDCan 2011 p. 7 (updated)"
.GPE_NEXT "What was a typical problem?"
.TITLE "Badly nested blocks and the Xo macro"
.MULB 6c 1n 6c 1n 9c
.SUBTITLE explicit
.DS
.ft CW
.ps -4
\&.Ao ao
\&.Bo bo
\&.No ac Ac
\&.No bc Bc
.sp
<ao [bo ac> bc]
.ft P
.ps +4
.DE
.MULN
.SUBTITLE implicit
.DS
.ft CW
.ps -4
\&.Aq aq Bo bo eol
\&.No bc Bc
.sp
<aq [bo eol> bc]
.ft P
.ps +4
.DE
.MULN
.SUBTITLE "in practice: .It Xo"
.DS
.ft CW
.ps -4
$ less `man -w find`
\&.It Xo
\&.Ic -exec Ar utility
\&.Op argument ...
\&.No ;
\&.Xc
.sp
-exec utility [argument ...] ;
.ft P
.ps +4
.DE
.MULE
.BL
.LI
Our first thought: deprecate this abomination, let people use:
.DS
.ft CW
.ps -4
\&.It Ic -exec Ar utility Oo argument ... Oc No ;
.ft P
.ps +4
.DE
.LI
But lots of manuals use ".It Xo" - historical argument limit
.LI
.GPE_EM "No way to change all manuals, everywhere,\
 or even the habits of authors..."
.LI
Reluctant implementation:
.br
Ao block contains the *whole* Bo block
.br
Bo block contains an Ao body-end element 
.LE
.GPE_TIME 90 "BSDCan 2011 p. 8 (updated)"
.GPE_NEXT "More real-world nuisances..."
.SUBTITLE "If ifs & ands were docs & mans..."
.TITLE "No way around some low-level roff requests."
.BL
.LI
Original design:
.BL
.LI
Forget about the complete bottom layer.
.LI
Only implement high-level mdoc(7) & man(7) macros. 
.LE
.LE
.GPE_MULB 15c
.BL
.LI
Didn't work out.
.BL
.LI
.GPE_EM "Real-world manuals use some low-level requests."
.LI
Realizing that was a slow, painful process.
.LI
We reluctantly edged in
.br
some low-level roff support.
.LI
Complexity was kept to the bare minimum. 
.LE
.LI
Both steps have proven to be just right:
.BL
.LI
Starting from the high level gave a clean design.
.LI
Roff reluctance prevented getting off track.
.LI
Surprise: Rather little impact in the end. 
.LE
.LE
.MULN
.PSPIC Images/TambakoZebra.eps
.GPE_SM "Very young zebra"
.GPE_SM "\(co 2009 Tambako @flickr (CC)"
.MULE
.GPE_TIME 80 "BSDCan 2011 p. 12"
.GPE_NEXT "What was done about low-level roff?"
.SUBTITLE "Desperation lead to success:"
.TITLE "How an afterthought improved the design."
.BL
.LI
Mandoc main program:
.BL
.LI
main loop to read input files
.LI
.GPE_EM "call the roff preprocessor on each line"
(not in original version)
.LI
push line after line into the parser backends
.LI
parsers look for high-level macros, e.g. mdoc(7)
.LI
call the formatting frontend on the resulting syntax tree 
.LE
.LI
Paradigmatic switch:
.BL
.LI
roff: expand high-level macros into low-level requests
.br
then pass the low-level requests into the formatters
.br
all structural info lost long before the main parser 
.LI
mandoc: first handle low-level requests (preprocessor)
.br
then pass the high-level code to the parsers
.br
structural info kept even when intermixed with low-level roff 
.LE
.LE
.GPE_TIME 90 "BSDCan 2011 p. 13"
.GPE_NEXT "2011 Conclusions"
.SUBTITLE "Done in 2011:"
.TITLE "Reached the stage: It just works."
.BL
.LI
To get there:
.BL
.LI
Re-implemented a small family of languages.
.LI
Turned the whole paradigm upside down.
.LI
Remained compatible where it matters.
.LI
Flouted compatibility that would just hinder. 
.LE
.LE
.sp
.GPE_MULB 13c
.BL
.LI
Other systems besides OpenBSD
.br
can now do the same, if they want.
.BL
.LI
\&... without much risk.
.LI
\&... without having to fear major surprises.
.LI
\&... getting support from us in case of need. 
.LE
.sp
Just mail us!
.LE
.MULN
.PSPIC Images/KrisVoordenFoal.eps
.GPE_SM "New Forest Foal"
.GPE_SM "\(co 2010 Krista van der Voorden @flickr (CC)"
.MULE
.GPE_TIME 50 "BSDCan 2011 p. 20"
.GPE_NEXT "2011 Conclusions"
.sp 2v
.SUBTITLE "Move fast!"
.TITLE "How a replacement project can succeed."
.GPE_MULB 14c
.BVL 1c
.LI "Quickly put your work to real-world use."
Do not let it rot in a corner of the OS. 
.LI "Keep moving fast, do not fear change."
But don't trap your users
.br
with incompatible changes. 
.LE
.P
And keep the balance:
.BL
.LI
No need to do it when it makes no difference.
.LI
You won't find users when you break behaviour.
.LI
You won't find bugs when you don't have users. 
.LE
.MULN
.PSPIC Images/GaryTannerFoal.eps 6c
.GPE_SM "Foals Just Wanna Have Fun"
.GPE_SM "\(co 2009 Gary Tanner @flickr (CC)"
.MULE
.GPE_TIME 30 "BSDCan 2011 p. 25"
.GPE_NEXT "2011 Conclusions"
.SUBTITLE "Bad patches triggering good ones:"
.TITLE "Preliminary code put in and ripped out again."
That may seem inefficient, but actually it's a perfectly sane approach:
.BL
.LI
The first implementation explores the feature.
.LI
The second implementation gets it right.
.LI
Just don't let the first one sprawl until it can't be ripped out any more. 
.LE
.SUBTITLE "This approach got used in at least five cases:"
.GPE_MULB 13c
.DS
2010 Mar 01 - May 14: end of sentence detection
2010 Apr 25 - May 19: roff conditionals
2010 Mar 01 - Sep 20: pod2man preamble
2010 Jun 16 - Jun 27: roff registers
2010 Oct 15 - Jan 04: tbl integration
.DE
.MULN
.PSPIC Images/TinyPackagesRoadside.eps
.GPE_SM "On the Road"
.GPE_SM "\(co 2010 tiny_packages @flickr (CC)"
.MULE
.GPE_TIME 90 "BSDCan 2011 p. 23 (slightly modified)"
.GPE_NEXT "2011 Conclusions"
.SUBTITLE "Newly designed:"
.TITLE "Clean implementation of dirty languages."
.BVL 1c
.LI "Started coding with the nice high-level stuff."
That gave us a very clean overall design. 
.LI "Edged in low-level ugliness later, where required."
Even while the tool was already in production! 
.LI "Only possible since we kept it small and simple."
Otherwise, such stunts would break the design. 
.LE
.GPE_MULB 12c
.SUBTITLE "Shun complexity!"
Three reasons why simplicity is key:
.AL
.LI
correctness
.LI
security
.LI
flexibility
.LE
.MULN
.PSPIC Images/MarkRobinsonQuantocks.eps
.GPE_SM "A Youngster on the Quantocks"
.GPE_SM "\(co 2009 Mark Robinson @flickr (CC)"
.MULE
.GPE_TIME 100 "BSDCan 2011 p. 24"
.GPE_SECTION ISSUES
.GPE_NEXT "What's the fuzz all about?"
.TITLE "American Fuzzy Lop inspecting mandoc"
.SUBTITLE "Fuzzer: Try to crash or hang a program by feeding it varying input."
.P
.GPE_MULB 8c
http://lcamtuf.coredump.cx/afl/
.P
"Compile-time instrumentation and genetic algorithms
to automatically discover clean, interesting test cases
that trigger new internal states."
.P
Goal: Full functional coverage.
.P
For mandoc, afl can be seeded with the extensive test suite.
.P
For mandoc, full functional coverage requires running afl continuously
for several days on modern PC hardware.
.MULN
.PSPIC Images/LithoniusAmericanFuzzyLop.eps
.GPE_SM "American Fuzzy Lop \(co 2008 Lithonius@wikimedia (PD)"
.MULE
.SUBTITLE "Run by Jonathan Gray (jsg@) repeatedly since November 21, 2014."
.ce
45 issues grand total \(-> Quantitative (but not statistical) analysis.
.GPE_TIME 90 "BSDCan 2015 p. 22"
.GPE_NEXT "What caused bugs?"
.TITLE "Results from the afl audit"
.GPE_MULB 12c
.BL
.LI
Causes of bugs:
.BL
.LI
1/3 violations of invariants
.LI
1/3 excessive complexity
.br
(in languages; in implementation)
.LI
1/9 missing input validation
.LI
1/9 buffer overflows
.LI
1/9 use after free
.LE
.LE
.MULN
.PSPIC Images/TudorCostacheBarrieSpiritcatcher.eps 4c
.GPE_SM "Spiritcatcher by Ron Baird (1986) in Barrie"
.GPE_SM "\(co 2009 Tudor Costache @wikimedia (CC)"
.MULE
.BL
.LI
Lessons:
.BL
.LI
Obviously most important point: Keep code small and simple.
.LI
Paying attention to the simplest sources of bugs tends to help
against the errors having the gravest consequences, even though may
no be the most numerous types of errors (in particular buffer and
integer overflows, use after free, missing input validation)
.LI
Explicitly specifying invariants and auditing for their correct
implementation is likely to catch a quantitatively important 
class of bugs (but a lot of work).
.LE
.LE
.GPE_TIME 120 "BSDCan 2015 pp. 23-25 (shortened)"
.GPE_NEXT "Which vulnerabilities were found?"
.TITLE "Security issues found in the web manual viewer"
.sp -1v
.nr Lsp_save \n[Lsp]
.nr Lsp 0.1v
.BL
.LI
Invalid configuration \(-> segfault:
.br
Fix: When there is no MAN_DIR or manpath.conf, log and exit.
.LI
Unvalidated PATH_INFO \(-> access to unrelated files, information disclosure:
.br
Fix: Reject absolute paths and ascenscion to the parent directory.
.LI
Unvalidated QUERY_STRING manpath \(-> info disclosure in error message:
.br
Fix: Validate the manpath up front using a whitelist.
.LI
Invalid characters in QUERY_STRING \(-> XSS:
.br
Fix: Restrict the character set of strings passed into html_alloc().
.LI
Roff escape sequences and quotes in manuals \(-> XSS:
.br
Fix: HTML-encode quotes and rendered escape sequences.
.LI
Choosing the right encoding is a tricky business...
.br
Some output needs HTML encoding, some URI encoding, some both.
.\" July 25
.LI
Crafted expensive regular expressions \(-> REDoS attacks:
.br
No full fix possible.
Mitigate by limiting CGI process execution time.
.\" August 21
.LE
.nr Lsp \n[Lsp_save]
.GPE_MULB 18c
.in 1c
.PSPIC Images/RyanTirMaggieLake.eps 17c
.MULN
.sp 1.5v
.S -4
.DS
Maggie Lake
Algonquin Park
Ontario, Canada
.sp
\(co 2011 Ryan Tir
@flickr (CC)
.DE
.S P
.MULE
.GPE_TIME 180 "BSDCan 2015 p. 18"
.GPE_NEXT "What about performance?"
.TITLE "Search and database performance summary"
.BL
.LI
With the old, plain text apropos(1), a simple search
took about 10 milliseconds on my notebook.
.LI
With the new, SQLite apropos(1), it is unavoidably slower
due to the SQL overhead and because the names are now separated
from the descriptions.  It now takes about 40 milliseconds.
.LI
However, the difference is of no practical relevance
even on moderately old hardware (like this notebook).
.LE
.sp 1v
.GPE_MULB 10c 0v
.BL
.LI
Base system database size grows from 250 kB to 900 kB (quick mode)
or about 3800 kB (fully featured more).
That is not a practical problem for any of our architectures.
.LI
During system builds, database build times are \m[red]reduced by
roughly a factor 3\m[] with respect to the old Perl makewhatis(8).
.LE
.MULN
.PSPIC Images/AvenueSpottedShag.eps
.ce 2
.GPE_SM "Spotted Shag / Parekareka (Phalacrocorax punctatus)"
.GPE_SM "(c) 2010 Avenue@wikimedia (CC)"
.MULE
.GPE_TIME 30 "BSDCan 2014 p.23"
.GPE_SECTION CONCLUSION
.GPE_NEXT "Conclusions"
.TITLE "Conclusions"
.SUBTITLE "What made the replacement project succeed"
.BL
.LI
Start with a clean design,
.br
sparingly add support for less clean,
historically grown features where needed.
.LI
Quickly go into production,
.br
profit from user feedback about bugs and missing functionality to improve.
.LI
Improve step by step aiming for complete functionality but avoid bloat:
.br
man(1) mdoc(7) man(7) tbl(7) eqn(7) apropos(1) man.cgi(8)
.LE
.GPE_MULB 12c
.sp
.SUBTITLE Priorities
.AL
.LI
correctness and security
.LI
compatibility
.LI
simplicity
.LI
performance
.LE
.P
\&... in that order.
.MULN
.PSPIC Images/PhilDavisSquirrel.eps 10c
.GPE_SM "Squirrel digging"
.GPE_SM "\(co 2010 Phil Davis @flickr (CC)"
.MULE
.GPE_TIME 60 "NYC*BUG 2015"
.GPE_NEXT "Let's summarize the status..."
.TITLE "Status summary"
.sp -1.5v
.BVL 1c
.LI "Fully integrated (mandoc, man, apropos, makewhatis)"
OpenBSD, Alpine Linux, Void Linux
.LI "Almost fully integrated (including apropos but without man)"
FreeBSD-current
.LI "Default formatter (but less powerful implementations of man and apropos)"
NetBSD, illumos
.LI "In the base system (but not used by default)"
FreeBSD 10, DragonFly BSD, Minix 3
.LI "Official packages exist"
FreeBSD 9, Arch Linux, pkgsrc
.LI "Unofficial packages of useful versions exist"
Slackware, Crux Linux, Mac OS X, MinGW
.LI "Outdated packages only, probably easy to update"
Debian, SUSE, Redhat and derivatives; IBM AIX, Cygwin
.LE
.sp -0.5v
.PSPIC Images/GBaranskiOttawaPanorama.eps
.GPE_SM "Ottawa (Ontario) seen from Gatineau (Quebec)\
 \(co 2009 G. Baranski @wikimedia (CC)"
.GPE_TIME 70 "BSDCan 2015 p. 39"
.GPE_NEXT "What shall we do in the future?"
.TITLE "Possible future directions"
.sp -1.5v
.SUBTITLE "Work in progress"
.BL
.LI
Improve pod2mdoc(1) and use it for LibreSSL (first mentioned: BSDCan 2011)
.LI
Unify parsers aiming for better roff(7) support
(first mentioned: BSDCan 2014)
.LI
Automatically detect unsupported source code (-Wunsupp) (NEW)
.LI
Delete most ln(1) links to manual pages (NEW)
.LI
texi2mdoc(1) to convert texinfo(1) documentation to mdoc(7) (NEW)
.LI
Help with man(7) to mdoc(7) conversions (first mentioned: BSDCan 2011);
docbook2mdoc(1) (first mentioned: BSDCan 2014)
.LE
.GPE_MULB 12c
.BL
.LI
Continue work on internal search
.br
and deep linking (NEW).
.LE
.SUBTITLE "Not yet started"
.BL
.LI
Support automatic semantic enrichment
.br
of Perl manuals with pod2mdoc(1).
.br
(first mentioned: EuroBSDCon 2014)
.LE
.MULN
.PSPIC Images/WladyslawGatineauMuseumOfHistory.eps
.GPE_SM "Canadian Museum of History, Gatineau"
.GPE_SM "\(co 2009 Wladyslaw@wikimedia (CC)"
.MULE
.GPE_TIME 80 "BSDCan 2015 p. 41 (updated)"
.GPE_SECTION THANKS
.GPE_NEXT "Who contributed?"
.TITLE Thanks!
.GPE_TIME 120 "NYC*BUG 2015"
.S -4
.BVL 1c
.LI "Cynthia Livingston (USENIX)"
for designing and implementing the mdoc(7) language
and translating all the BSD manual pages to it
.LI "Kristaps Dzonsons (bsd.lv)"
for writing mandoc
.LI "J\(:org Sonnenberger (NetBSD)"
for important code contributions and
for hosting an excellent mandoc hackathon at BEC.de
.LI "Marc Espie (OpenBSD)"
for OpenBSD ports integration, lots of important feedback,
and for writing the ohash library
.LI "Jonathan Gray (OpenBSD)"
for extensive testing with afl resulting in over 40 bug reports
.LI "S\('ebastien Marie"
for a complete man.cgi(8) security audit
.LI "Todd C. Miller (OpenBSD & sudo)"
for feedback and multiple patches to the mdoc-to-man converter
.LI "Jason McIntyre (OpenBSD)"
for excellently and tirelessly maintaining our manuals,
for helping with countless bug reports, and
for discussing countless questions regarding mdoc(7)
.LI "Theo de Raadt (OpenBSD)"
for inviting Kristaps and getting mandoc imported.
(Otherwise, i might have missed it.)
.br
for ongoing encouragement, in particular to make OpenBSD developers and
users our guinea pigs. (None complained, they seemed to enjoy it.) 
.br
for many bug reports and some patches
.LI "Anthony J. Bentley (OpenBSD)"
for porting related software to OpenBSD, many bug reports,
and some source code patches
.LE
.GPE_MULB 13c
.BVL 1c
.LI "Jeremy Evans (OpenBSD)"
for crucial help with SQLite database optimization
.LI "Matthieu Herrb, Todd Fries, Miod Vallat (OpenBSD)"
for help with Xenocara integration
.LI "Christian Weisgerber (OpenBSD)"
for continuous work on mandoc issues in OpenBSD ports
.LI "Stuart Henderson (OpenBSD)"
for help with large numbers of porting issues
.LI "Pascal Stumpf (OpenBSD)"
for repeated help with difficult groff porting issues
.LI "Peter Hessler (OpenBSD)"
for help with the regression infrastructure
.LI "Thomas Klausner (NetBSD)"
for NetBSD and pkgsrc porting work
.br
and lots of feedback and release testing
.LI "Baptiste Daroussin and Ulrich Sp\(:orlein (FreeBSD)"
for FreeBSD porting and many bug reports
.LI "Werner Lemberg (GNU troff)"
for tireless help with groff-mandoc synchronization
.LE
.MULN
.PSPIC Images/MarkJoblingRobin.eps
.S +4
.GPE_SM "Robin / Toutouwai (Petroica australis)"
.GPE_SM "\(co 2007 Mark Jobling @wikimedia (PD)"
.S -4
.MULE
.BVL 1c
.LI "Natanael Copa, Paul Onyschuk (Alpine), Svyatoslav Mishyn (Crux),\
 Jesse Adams (Arch), D\('aniel L\('evai (Slackware)"
for porting mandoc(1) to Linux and providing feedback
.LI "Garrett D'Amore, Yuri Pankov (IllumOS),\
 Matthias Scheler (Solaris), Ben Gras (Minix 3)"
for porting mandoc(1) to Non-BSD systems and providing feedback
.LE
.S +4
.GPE_NEXT "Who contributed?"
.S -2
.SUBTITLE "For source code patches:"
Franco Fichtner (DragonFly), Christos Zoulas, Tsugutomo Enami (NetBSD),
Daniel Dickman, Doug Hogan, Ted Unangst (OpenBSD), Martin Natano
.SUBTITLE "For bug reports and useful suggestions and discussions:"
Alexander Bluhm, Antoine Jacoutot, Bob Beck, Brad Smith, Bret Lambert,
Brian Callahan, Bryan Steele,
David Coppa, David Gwynne, Dmitrij D. Czarkoff, Edd Barrett, Florian Obser,
Giovanni Becchis, Gleydson Soares, Henning Brauer, Ian Darwin, Igor Sobrado,
Janne\ Johansson, Jasper Lievisse Adriaanse, J\('er\('emie Courr\(`eges-Anglas,
Jonathan Gray, Juan\ Francisco Cantero Hurtado,
Kenji Aoyama, Kenneth R. Westerback, Kurt Miller, Landry\ Breuil,
Martin Pieuchot, Matthew Dempsky, Matthias Kilian,
Nicholas Marriott, Nick\ Holland, Nigel Taylor, Okan Demirmen,
Paul Irofti, Paul de Weerd, Philip Guenther,
Remi\ Pointel, Reyk Fl\(:oter, Stefan Sperling,
Thordur Bjoernsson, Vadim Zhukov (OpenBSD)
.P
Abhinav Upadhyay, David Holland, Nicolas Joly, Havard Eidnes,
Jonathan Perkin (NetBSD),
Kurt Jaeger, Pedro Giffuni (FreeBSD),
Antonio Huete Jimenez, Sascha Wildner (DragonFly),
Michael Dexter (bsd.lv),
Carsten Kunze (Heirloom Doctools),
Alexis Hildebrandt (Homebrew)
.P
Alessandro de Laurenzis, Andreas V\(:ogele, 
Chris Bennett, Chris Hettrick, Christian\ Neukirchen, David Hill, David Levine,
Fabian Raetz, Frantisek\ Holop, Guy Harris, Igor\ Zinovik,
James Jerkins, Jan Stary, J\(:orn Clausen, Justin\ Haynes,
Marcus Merighi, Markus\ Waldeck, Maxim Belooussov,
Michel Jansens, Michael Reed, Mike Small, Mikolaj\ Kucharski,
Patrick Keshishian, Ryan Flannery, Steffen Nurpmeso,
Theo B\(:uhler, Tim\ van\ der\ Molen, Tristan Le Guern, Will Backman
.S +2
.GPE_TIME 60 "NYC*BUG 2015 (updated)"
.GPE_NEXT "Where did the images come from?"
.sp -0.2v
.SUBTITLE "Thank you for sharing your pictures!"
.sp -0.7v
.S -9
.DS
http://www.flickr.com/photos/tomkoadam/4778126822\
 Adam Tomk\('o: Csik\('o \(en Foal (by-nc-nd)
http://www.flickr.com/photos/whereisbrent/461055143\
 Brent Barrett: Kea juvenile (by-nc-nd)
http://2014.eurobsdcon.org/wp-content/uploads/2014/06/BSDSofiaForWeb.png\
 Alica Dimitrova: Sofia BSD Mascot
Cynthia Livingston's off-track thoroughbred "Bedifferent"\
 (private communication, used with permission)
https://www.flickr.com/photos/14020964@N02/7373733864/\
 Lezumbalaberenjena: Black Lake, Gatineau Park, Quebec (by-nc-nd)
http://commons.wikimedia.org/wiki/File:StGeorgeRotundaSofia.JPG\
 Preslav: Rotonda Sveti Georgi, Sophia (pd)
http://commons.wikimedia.org/wiki/File:Maliovitsa_54072.jpg\
 Stelian Kasabov: Maljovitsa 2729m, Rila (pd)
http://www.flickr.com/photos/everexplore/8572686541\
 Kiril Rusev: Kutelo 2908m and Vihren 2914m, Pirin (by-nc-nd)
http://www.mckusick.com/beastie/shirts/bsdunix.html\
 USENIX: 4.2BSD Beastie
http://www.flickr.com/photos/29954808@N00/1300190844\
 57Andrew: Rock Wren (by)
http://commons.wikimedia.org/wiki/File:Tokoeka.jpg\
 Glen Fergus: Southern Kiwi (by-sa)
http://commons.wikimedia.org/wiki/File:Silvereye.jpg\
 J. J. Harrison: Silvereye (by-sa)
http://www.flickr.com/photos/dougtone/5806473660/\
 Doug Kerr: Bonnechere Museum, Eganville, Renfrew County, Ontario (by-sa)
http://commons.wikimedia.org/wiki/File:Red_fox_kit_2_%28Vulpes_vulpes%29.jpg\
 Charlesjsharp: Red fox kit (by-sa)
http://www.flickr.com/photos/reiver/3821502286/\
 Mac Armstrong: Moorside cottage, Mackenzie King Estate,\
 Gatineau Park, Quebec (by-sa)
http://commons.wikimedia.org/wiki/File:Aerial_-_Wasaga_Beach,_Ontario_from_SW_01_-_white_balanced_%289656223451%29.jpg\
 Joe Mabel: Wasaga Beach (by-sa)
http://www.flickr.com/photos/jimmy-jay/4672901414\
 Jeff Burcher: Horse Fly Portrait (by-nc-nd)
http://www.flickr.com/photos/docnz/8528275645\
 NZ DOC: Kakapo (by-nc-sa)
http://www.flickr.com/photos/dexxus/5003010775/\
 Paul Bica: Humber Bridge, Toronto, Ontario (by)
http://www.flickr.com/photos/tambako/3578468294\
 Tambako: Very young zebra (by-nd)
http://www.flickr.com/photos/kristavandervoorden/4737488285\
 Krista van der Voorden: New Forest Foal (CC)
http://www.flickr.com/photos/gazzat/3495392530\
 Gary Tanner: Foals Just Wanna Have Fun (by-na-sa)
http://www.flickr.com/photos/tiny_packages/5045038219\
 tiny_packages: On the road (by-nc-nd)
http://www.flickr.com/photos/66176388@N00/3436935367\
 Mark Robinson: A Youngster on the Quantocks (by-nc)
http://commons.wikimedia.org/wiki/File:Rabbit_american_fuzzy_lop_buck_white.jpg Lithonius: American Fuzzy Lop rabbit (pd)
http://commons.wikimedia.org/wiki/File:Spiritcatcher_barrie_wide.jpg\
 Tudor Costache: Spiritcatcher by Ron Baird, Barrie, Ontario (by)
http://www.flickr.com/photos/ryan_tir/6232749531/\
 Ryan Tir: Maggie Lake, Algonquin Park, Ontario (by)
http://commons.wikimedia.org/wiki/File:Spotted_Shag_%28Phalacrocorax_punctatus%29_in_flight_2.jpg Avenue: Spotted Shag (by-sa)
http://www.flickr.com/photos/eastsidephil/4452171897\
 Phil Davis: Squirrel digging (by-nc-sa)
http://commons.wikimedia.org/wiki/File:Canada_Ottawa_Panorama.jpg\
 G. Baranski: Ottawa Panorama (by-sa)
http://commons.wikimedia.org/wiki/File:Gatineau_-_QC_-_Museum_of_Civilisation.jpg\
 Wladyslaw: Canadian Museum of History, Gatineau (by-sa)
http://commons.wikimedia.org/wiki/File:070308_Stewart_Island_robin_on_Ulva.jpg\
 Mark Jobling: NZ Robin (pd)
.DE
.S P
.sp -1v
.PSPIC Images/RaulHeinrichAlgonquinTeaLake.eps
.GPE_SM "North Tea Lake, Algonquin Provincial Park\
 \(co 2008 Raul Heinrich @wikimedia (CC)"
.GPE_TIME 10 "NYC*BUG 2015"
.ds gpe_next What would you like to ask?