<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">% 
% \iffalse
%&lt;*driver&gt;
\documentclass{tcldoc}

\usepackage{array}


% Logos:
% 
\DeclareFontFamily{U}{logo}{}
\DeclareFontShape{U}{logo}{m}{n}{
  &lt;-9&gt;   logo8
  &lt;9-10&gt; logo9
  &lt;10-&gt;  logo10
}{}
\DeclareFontShape{U}{logo}{bx}{n}{&lt;-&gt; logobf10}{}
\DeclareFontShape{U}{logo}{b}{n}{&lt;-&gt; ssub * logo/bx/n}{}
\DeclareFontShape{U}{logo}{m}{sl}{&lt;-&gt; logosl10}{}
\DeclareFontShape{U}{logo}{m}{it}{&lt;-&gt; ssub * logo/m/sl}{}

% \DeclareRobustCommand\META{{%
%    \fontencoding{U}\fontfamily{logo}\selectfont META%
% }}
\DeclareRobustCommand\MF{%
   {\fontencoding{U}\fontfamily{logo}\selectfont META}\-%
   {\fontencoding{U}\fontfamily{logo}\selectfont FONT}%
}
\DeclareRobustCommand\MP{%
   {\fontencoding{U}\fontfamily{logo}\selectfont META}\-%
   {\fontencoding{U}\fontfamily{logo}\selectfont POST}%
}
\DeclareRobustCommand\PS{PostScript}
\DeclareRobustCommand\TOne{Type~1}
\DeclareRobustCommand\package[1]{\textsf{#1}}


% Useful environments:
% 
% \begin{environment}{isyntax}
%   |isyntax| is for ``informal'' syntax specifications, such as
%   \begin{quote}
%     A \meta{varchar command} can be one of\\
%     \begin{quote}
%       |\vartop|\marg{glyph}\\
%       |\varmid|\marg{glyph}\\
%       |\varbot|\marg{glyph}\\
%       |\varrep|\marg{glyph}
%     \end{quote}
%   \end{quote}
%   The implementation is based on that of the \texttt{verse} 
%   environment.It usually works just as fine with \texttt{quote}, but 
%   \texttt{isyntax} handles breaking long lines better (and more 
%   elegantly).
%   \changes{1.37}{2000/05/14}{Increased \cs{rightskip} stretchability 
%      and added setting of \cs{linepenalty}. (LH)}
%    \begin{macrocode}
\makeatletter
\newenvironment{isyntax}{%
   \let\\\@centercr
   \list{}{%
      \itemsep \z@
      \itemindent -1.5em%
      \listparindent \itemindent
      \advance \leftmargin 1.5em%
   }%
   \advance \rightskip \z@\@plus0.7\linewidth \relax
   \linepenalty=100\relax
   \item\relax
}{\endlist}
\makeatother
%    \end{macrocode}
% \end{environment}


% Tabular notes:
\makeatletter
\newcommand\tabnotemark[1]{%
   \leavevmode
   \ifhmode\edef\@x@sf{\the\spacefactor}\nobreak\fi
   \hbox{\@textsuperscript{%
      \normalfont\itshape
      \count@=#1\relax\@alph\count@
   }}%
   \ifhmode\spacefactor\@x@sf\fi
   \relax
}
\newenvironment{tabnotes}{%
   \list{\the\c@mpfootnote}{%
      \usecounter{mpfootnote}%
      \def\makelabel##1{\hfil\tabnotemark{##1}}%
      \normalfont\footnotesize
      \setlength\leftmargin{0pt}%
      \setlength\labelsep{0pt}%
      \setlength\labelwidth{1.8em}%
      \setlength\itemindent{\labelwidth}%
      \setlength\listparindent{1em}%
      \setlength\parsep{0pt plus 1pt}%
      \setlength\itemsep{\parsep}%
      \setlength\topsep{\footnotesep}%
   }%
}{\endlist}
\makeatother


\CodelineIndex

\title{\texttt{pstokens.dtx}}
\author{Lars Hellstr\"om}
\date{Work in progress}
\begin{document}
\maketitle

\DocInput{pstokens.dtx}

\end{document}
%&lt;/driver&gt;
% \fi
% 
% \begin{abstract}
%   \package{pstokens} is a collection of utility routines for 
%   \Tcllogo\ programs that need to view a file as a sequence of 
%   \PS\ tokens rather than as a sequence of characters or lines.
% \end{abstract}
% 
% 
% \setnamespace{pstokens}
% 
% First of all, the \texttt{pstokens} namespace must be defined.
% \begin{tcl}
namespace eval pstokens {}
% \end{tcl}
% 
% \section{Reading and writing tokens}
% 
% This section contains the routines for reading a text file as a 
% sequence of tokens (tokenizing) and writing a sequence of tokens 
% to a text file (detokenizing). 
% 
% 
% \subsection{Data structures for tokens}
% 
% A \emph{token} is a \Tcllogo\ list with the format
% \begin{quote}
%   \word{type} \word{data}\regstar
% \end{quote}
% The interpretation of the \word{data} depends on what the \word{type} 
% is, as specified below. Some of the token types only occur internally 
% in the tokenizer and some others have no meaning to a \PS\ interpreter, 
% but this organisation seems to be the most convenient here.
% 
% A primary goal for the format of tokens is that if a sequence of 
% tokens are read from one file and written to another then the only 
% things that may have changed should be the whitespace between the 
% tokens, not the tokens themselves. Therefore the tokenizer generally 
% avoids interpreting the tokens any more than necessary; 
% interpretation of tokens is instead handled in the next section.
% 
% The token types that occur in \package{pstokens} are
% \begin{description}
%   \item[\texttt{number}, \texttt{radixnumber}]
%     \describestring[token type]{number}
%     \describestring[token type]{radixnumber}
%     A number (integer or real), and radix number, respectively. There 
%     is one \word{data} item, which contains the string that 
%     represented the number in the tokenized text.
%   \item[\texttt{string}, \texttt{hexstring}, \texttt{base-85}]
%     \describestring[token type]{string}
%     \describestring[token type]{hexstring}
%     \describestring[token type]{base-85}
%     A parenthesis-delimited string, hexadecimal string, and ASCII 
%     base-85 string, respectively. There is one \word{data} item, 
%     which contains the string (minus delimiters) that represented 
%     the string in the tokenized text.
%   \item[\texttt{executable}, \texttt{literal}, \texttt{immediate}]
%     \describestring[token type]{executable}
%     \describestring[token type]{literal}
%     \describestring[token type]{immediate}
%     An executable (normal), literal, and immediately evaluated name 
%     respectively. There is one \word{data} item, which simply contains 
%     the name as a \Tcllogo\ string, not including any initial slashes.
%   \item[\texttt{procedure}]
%     \describestring[token type]{procedure}
%     An executionable array. The data is simply the sequence of tokens 
%     in the array.
%   \item[\texttt{comment}]
%     \describestring[token type]{comment}
%     This is not a \PS\ token at all, but a representation of a 
%     comment. The reason comments are tokenized is that they often 
%     contain interesting information (especially structured comments).
%     The structure of this kind of token is
%     \begin{quote}
%       |comment| \word{head} \word{tail}
%     \end{quote}
%     The \word{head} is the part of the line which is after the |%| 
%     that started the comment and before the first whitespace character 
%     after it. The \word{tail} is the rest of the line (including the 
%     whitespace character, if there is one). Any of \word{head} and 
%     \word{tail} may be empty. The structured comment
%     \begin{quote}
%       |%%Pages: 2|
%     \end{quote}
%     would have \word{head} equal to \verb*"%Pages:" and \word{tail} 
%     equal to \verb*" 2".
%   \item[\texttt{eof}]
%     \describestring[token type]{eof}
%     This is a pseudotoken which denotes that the end of the file was 
%     reached while looking for the next token. There structure is
%     \begin{quote}
%       |eof| \word{token}\regopt
%     \end{quote}
%     If the \word{token} is present then it is a composite token which 
%     was being read when the file unexpectedly ended. It is an error 
%     if the \word{token} is present.
%   \item[\texttt{special}]
%     \describestring[token type]{special}
%     This token type can be used to extend the token type syntax 
%     without having to change the actual routines for reading and 
%     writing tokens. The structure of a |special| token is
%     \begin{quote}
%       |special| \word{token} \word{extra data}\regstar
%     \end{quote}
%     If a |special| token gets sent to the token writer then it will be 
%     treated as the token \word{token} (which usually has another 
%     type), but other routines may treat it differently based on the 
%     information in the \word{extra data}.
%   \item[\texttt{error}]
%     \describestring[token type]{error}
%     A token of this type is returned from the tokenizer whenever it 
%     encounters anything which is syntactically incorrect. The 
%     general structure of this kind of token is
%     \begin{quote}
%       |error| \word{error type} \word{data}\regstar
%     \end{quote}
%     where the \word{error type} more precisely specifies the error. 
%     So far, the following error types are used
%     \begin{quote}
%       |error| |extra| \word{string} \word{line no.}\\
%       |error| |missing| \word{string} \word{line no.}\\
%       |error| |missing/extra| \word{m-string} \word{e-string} 
%       \word{line no.}
%     \end{quote}
%     |extra| means that there is an occurence of \word{string} which 
%     shouldn't have been there, whereas |missing| means that there 
%     should have been a \word{string} there, but isn't. |missing/extra| 
%     is the combination that there either should have been an 
%     \word{m-string} there, but isn't, or there is an \word{e-string} 
%     there, but there shouldn't be.
%     \word{line no.} is the line on which the error was encountered.
% \end{description}
% 
% 
% \subsection{Reading tokens}
% 
% \begin{arrayvar}{input_line}[file-id]
% \begin{arrayvar}{input_line_no}[file-id]
%   Since the |gets| procedure always reads an entire line (which 
%   usually contain several tokens) each time it is used, it is 
%   necessary to store the most recently read line somewhere. This is 
%   the purpose of the |input_line| array, which is indexed by the file 
%   id. In general, text is removed from the entires in |input_line| as 
%   it is tokenized.
%   
%   The |input_line_no| array stores the number of the line in the 
%   corresponding |input_line| entry. The first line in a file has 
%   number one.
% \end{arrayvar}\end{arrayvar}
% 
% \begin{proc}{open_input}
% \begin{proc}{close_input}
%   These procedures are wrappers around \Tcllogo's |open| and |close| 
%   procedures respectively which additionally initialize and unset the 
%   relevant entry of the |input_line| and |input_line_no| arrays. Thus 
%   their syntaxes are
%   \begin{quote}
%     |pstokens::open_input| \word{file-name} \word{access}\regopt\ 
%     \word{permissions}\regopt\\
%     |pstokens::close_input| \word{file-id}
%   \end{quote}
%   and they return the same things as |open| and |close| respectively.
%   
%   A possible development of tokenizer is to implement filtered input, 
%   in which case more support data structures would have to be 
%   initialized, but then the |open_input| and |close_input| procedures 
%   would be redefined to cope with this.
%   \begin{tcl}
%&lt;*tokenio&gt;
proc pstokens::open_input {args} {
   set res [eval open $args]
   global pstokens::input_line pstokens::input_line_no
   set pstokens::input_line($res) ""
   set pstokens::input_line_no($res) 0
   return $res
}
% \medskip
proc pstokens::close_input {file_id} {
   global pstokens::input_line pstokens::input_line_no
   unset pstokens::input_line($file_id)
   unset pstokens::input_line_no($file_id)
   close $file_id
}
%   \end{tcl}
% \end{proc}\end{proc}
% 
% The tokenizer has a couple of extra internal token types for things 
% which later become parts of other tokens.
% \begin{description}
%   \item[\texttt{beginproc}, \texttt{endproc}]
%     \describestring[token type]{beginproc}
%     \describestring[token type]{endproc}
%     These are the |{| and |}| at the beginning and end respectively of 
%     a \PS\ procedure. There is no \word{data}.
%   \item[\texttt{beginstring}, \texttt{beginhex}, \texttt{beginbase85}]
%     \describestring[token type]{beginstring}
%     \describestring[token type]{beginhex}
%     \describestring[token type]{beginbase85}
%     These token types are the |(|, |&lt;|, and |&lt;~| at the beginning of 
%     a \PS\ literal, hex, and base-85 string respectively. There is no 
%     \word{data}.
% \end{description}
% 
% 
% \begin{proc}{get_primitive_token}
%   The |get_primitive_token| procedure returns the next primitive 
%   token in the input file. It takes the file id of the source to read 
%   from as its only argument.
%   
%   A \emph{primitive token} is a smallest syntactic unit in the \PS\ 
%   file. Smallest syntactic units are characterized by that the 
%   insertion of a space anywhere in them changes its meaning. One 
%   exception is however the left parenthesis characters; it is 
%   convenient to include |(| among the primitive tokens even though 
%   inserting a space after it will change the meaning of 
%   the corresponding token.
%   
%   The first step in |get_primitive_token| is to read ahead until some 
%   non-whitespace (or the end of the file) is encountered.
%   \begin{tcl}
proc pstokens::get_primitive_token {file} {
   global pstokens::input_line pstokens::input_line_no
   set pstokens::input_line($file)\
      [string trimleft [set pstokens::input_line($file)] \t\n\f\r\ ]
   while {[string length [set pstokens::input_line($file)]]==0} {
      if {[gets $file pstokens::input_line($file)]==-1} then {
         return {eof}
      }
      incr pstokens::input_line_no($file)
      set pstokens::input_line($file)\
         [string trimleft [set pstokens::input_line($file)] \t\n\f\r\ ]
   }
%   \end{tcl}
%   The second step is to isolate the part of the text that constitutes 
%   the next token. This is done using regular expressions, but as these
%   get rather complicated it seems best to explain how they are 
%   constructed. The ASCII characters null (|\x00|), tab (|\x09|), line 
%   feed (|\x0a|), form feed (|\x0c|), carriage return (|\x0d|), and 
%   space (|\x20|) count as whitespace and delimit tokens, but null is 
%   currently ignored because \Tcllogo\ before v\,8 cannot keep them in 
%   strings anyway. The ASCII characters |(|, |)|, |&lt;|, |&gt;|, |[|, |]|, 
%   |{|, |}|, |/|, and |%| also delimit tokens, and therefore the 
%   regular expression for a \PS\ name is
%   \begin{quote}
%     |(/?/?)([^][()&lt;&gt;{}/%|\meta{tab}\meta{LF}\meta{FF}\meta{CR}^^A
%     \verb*" ]+)|//?"
%   \end{quote}
%   (Note that the first right bracket above does not end a character 
%   range, but is part of the list of characters which are exculded 
%   from it. Also note that |/| and |//| are valid \PS\ tokens.)
%   The number of initial slashes determine whether the name is 
%   \texttt{executable}, \texttt{literal}, or \texttt{immediate}.
%   
%   The executable tokens are however somewhat tricky, because the above 
%   regular expression will also match all numbers. An \emph{integer} is 
%   any token which matches the regular expression \verb"[+-]?[0-9]+", 
%   whereas a \emph{real number} is anything else which matches the 
%   regular expression
%   \begin{quote}
%     \verb"[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?"
%   \end{quote}
%   and a \emph{radix number} is anything which matches the regular 
%   expression\footnote{Actually there is also the restriction that 
%   none of the characters after the \texttt{\#} may represent a digit 
%   greater than or equal to the decimal number before the \texttt{\#}, 
%   but I'm ignoring that, for now.}
%   \begin{quote}
%     |[0-9][0-9]?#[0-9A-Za-z]+|
%   \end{quote}
%   \begin{tcl}
   if {[regexp "^(/?/?)(\[^\]\[()&lt;&gt;\{\}/%\t\n\f\r\ \]+)|^//?"\
      [set pstokens::input_line($file)] tokenstr slashes name]}\
   then {
      if {[regexp\
            {^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$}\
            $tokenstr]}\
      then {
         set token [list number $tokenstr]
      } elseif {[regexp {^[0-9][0-9]?#[0-9A-Za-z]+$} $tokenstr]} then {
         set token [list radixnumber $tokenstr]
      } else {
         if {"$tokenstr"=="/" || "$tokenstr"=="//"}\
         then {set slashes $tokenstr}
         set token [list [lindex {executable literal immediate}\
            [string length $slashes]] $name]
      }
   }\
%   \end{tcl}
%   There are also a couple of primitive tokens which consist of 
%   non-whitespace delimiting characters, namely
%   \begin{quote}
%     |{|, |}|, |&lt;|, |&lt;~|, |&lt;&lt;|, |&gt;&gt;|, |[|, |]|, |(|, 
%   \end{quote}
%   The right parenthesis and greater than characters does not appear in 
%   the list because they may not appear outside a string. Finally 
%   there are the comments, which have the regular expression 
%   \begin{quote}
%     |^%([^|\meta{tab}\meta{LF}\meta{FF}\meta{CR}\verb*" ]*)(.*)$"
%   \end{quote}
%   Here the first parenthesis is the head of the comment and the second 
%   is the tail.
%   \begin{tcl}
   else {
      switch -regexp -- [set pstokens::input_line($file)] {
         ^\{ {set token beginproc ; set tokenstr \{}
         ^\} {set token endproc ; set tokenstr \}}
         ^\\\[ {set token {executable [} ; set tokenstr \[}
         ^\\\] {set token {executable ]} ; set tokenstr \]}
         ^&lt; {set token beginhex ; set tokenstr &lt;}
         ^&lt;~ {set token beginbase85 ; set tokenstr &lt;~}
         ^&lt;&lt; {set token {executable &lt;&lt;} ; set tokenstr &lt;&lt;}
         ^&gt;&gt; {set token {executable &gt;&gt;} ; set tokenstr &gt;&gt;}
         ^\\( {set token beginstring ; set tokenstr (}
         ^% {
            regexp "^%(\[^\t\n\f\r\ \]*)(.*)\$"\
              [set pstokens::input_line($file)] tokenstr head tail
            set token [list comment $head $tail]
         }
         default {
%   \end{tcl}
%   But if none of the above matched the next character must be 
%   erroneous.
%   \begin{tcl}
            set tokenstr\
               [string index [set pstokens::input_line($file)] 0]
            set token [list error extra $tokenstr\
               [set pstokens::input_line_no($file)] ]
         }
      }
   }
   set pstokens::input_line($file)\
      [string range [set pstokens::input_line($file)]\
         [string length $tokenstr] end]
   return $token
}
%   \end{tcl}
%   
% \end{proc}
% 
% \begin{proc}{get_string}
%   The |get_string| procedure reads a \PS\ literal string and converts 
%   it to a \texttt{string} token, which it returns. The only argument 
%   is the file id of the input file. It is assumed that the next thing 
%   on the input file line is the character after the left parenthesis 
%   that started the string.
%   
%   The only conversion of the data that |get_string| performs is that 
%   escaped newlines are skipped and non-escaped newlines are replaced 
%   by |\n| escapes. If the end-of-file is encountered while reading a 
%   string then the necessary number of right parentheses are added to 
%   the string and the resulting string token is built into an |eof| 
%   token, which is returned.
%   
%   The local variable |level| keeps track of the parenthesis nesting 
%   level, the local variable |data| is the string read so far, and the 
%   local variable is a flag which is |1| if the next character should 
%   be escaped and |0| otherwise.
%   \begin{tcl}
proc pstokens::get_string {file} {
   upvar #0 pstokens::input_line line
   global pstokens::input_line_no
   set level 1
   set data ""
   set escaped 0
   while {$level&gt;0} {
      while {$level&gt;0 &amp;&amp; [string length $line($file)]&gt;0} {
         set c [string index $line($file) 0]
         if {$escaped} then {
            append data $c
            set escaped 0
         } else {
            switch -exact -- $c {
               ( {incr level ; append data $c}
               ) {
                  incr level -1
                  if {$level&gt;0} then {append data $c}
               }
               \\ {set escaped 1 ; append data $c}
               default {append data $c}
            }
         }
         set line($file) [string range $line($file) 1 end]
      }
      if {$level&gt;0} then {
         if {!$escaped} then {append data \\n}
         set escaped 0
         if {[gets $file line($file)]==-1} then {
            while {$level&gt;1} {append data ) ; incr level -1}
            return [list eof [list string $data]]
         }
         incr pstokens::input_line_no($file)
      }
   }
   list string $data
}
%   \end{tcl}
% \end{proc}
% 
% 
% \begin{proc}{get_hexstring}
%   The |get_hexstring| procedure reads a \PS\ hexadecimal string and 
%   converts it to a \texttt{hexstring} token, which it returns. The 
%   only argument is the file id of the input file. It is assumed that 
%   the next thing on the input file line is the character after the 
%   less than character that started the string.
%   
%   The only conversion of the data that |get_hexstring| performs is that 
%   embedded whitespace is removed. If a character which is neither 
%   whitespace, a hexadecimal digit, or the closing |&gt;| is encountered 
%   then an \texttt{error} token is returned and the text read is put 
%   back into the |input_line| buffert (followed by an |&gt;| which is 
%   inserted to terminate the string. If the end-of-file is 
%   encountered while reading a string then what has been read so far 
%   is returned as a hexstring token built into an |eof| token.
%   \begin{tcl}
proc pstokens::get_hexstring {file} {
   upvar #0 pstokens::input_line line
   global pstokens::input_line_no
   set data ""
   while 1 {
      while {[string length $line($file)]==0} {
         if {[gets $file line($file)]==-1} then {
            return [list eof [list hexstring $data]]
         }
         incr pstokens::input_line_no($file)
      }
      set c [string index $line($file) 0]
      set line($file) [string range $line($file) 1 end]
      switch -regexp -- $c {
         {[0-9a-fA-F]} {append data $c}
         \[\t\n\f\r\ \] {}
         &gt; {break}
         default {
            set line($file) &lt;$data&gt;$c$line($file)
            return [list error missing/extra &gt; $c\
               [set pstokens::input_line_no($file)]]
         }
      }
   }
   list hexstring $data
}
%   \end{tcl}
% \end{proc}
% 
% 
% \begin{proc}{get_base85string}
%   The |get_base85string| procedure reads a \PS\ base-85 string and 
%   converts it to a \texttt{base-85} token, which it returns. The 
%   only argument is the file id of the input file. It is assumed that 
%   the next thing on the input file line is the character after the 
%   |&lt;~| characters that started the string.
%   
%   The only conversion of the data that |get_base85string| performs is 
%   that embedded whitespace is removed. If a character which is neither 
%   whitespace, a base-85 digit, the letter |z|, or the closing |~| is 
%   encountered then an \texttt{error} token is returned and the text 
%   read is put back into the |input_line| buffert (followed by a |~&gt;| 
%   that terminates the string). If the end-of-file is encountered while 
%   reading a string then what has been read so far is returned as a 
%   base-85 string token built into an |eof| token.
%   \begin{tcl}
proc pstokens::get_base85string {file} {
   upvar #0 pstokens::input_line line pstokens::input_line_no line_no
   set data ""
   while 1 {
      while {[string length $line($file)]==0} {
         if {[gets $file line($file)]==-1} then {
            return [list eof [list base-85 $data]]
         }
         incr line_no($file)
      }
      set c [string index $line($file) 0]
      set line($file) [string range $line($file) 1 end]
      switch -regexp -- $c {
         {[!-uz]} {append data $c}
         \[\t\n\f\r\ \] {}
         ~ {break}
         default {
            set line($file) &lt;~$data~&gt;$c$line($file)
            return [list error missing/extra ~&gt; $c $line_no($file)]
         }
      }
   }
%   \end{tcl}
%   If the character after the closing |~| is not a |&gt;| then the entire 
%   string is put back in the |input_line| buffert and a |missing| error 
%   is issued.
%   \begin{tcl}
   if {[string length $line($file)]&gt;0 &amp;&amp;\
         "[string index $line($file) 0]"=="&gt;"} then {
      set line($file) [string range $line($file) 1 end]
      list base-85 $data
   } else {
      set line($file) &lt;~$data~&gt;$line($file)
      list error missing &gt; $line_no($file)
   }
}
%   \end{tcl}
% \end{proc}
% 
% 
% \begin{proc}{get_token}
% \begin{proc}{get_token_rek}
%   The |get_token| procedure has the syntax
%   \begin{quote}
%     |pstokens::get_token| \word{file-id}
%   \end{quote}
%   and returns the next \PS\ token in the \word{file-id} file.
%   
%   Most of the work is however done by the |get_token_rek| procedure, 
%   which is a recursive form of |get_token|. |get_token_rek| handles 
%   converting the various |begin|\textellipsis\ token types to their 
%   complete forms, and it calls itself recursively to get the tokens 
%   that form a procedure. |get_token| only checks that there are no 
%   lone |endproc| tokens.
%   \begin{tcl}
proc pstokens::get_token {file} {
   set token [pstokens::get_token_rek $file]
   if {"[lindex $token 0]" == "endproc"} then {
      global pstokens::input_line_no
      set token [list error extra \} [set pstokens::input_line_no($file)]]
   }
   return $token
}
% \medskip
proc pstokens::get_token_rek {file} {
   set token [pstokens::get_primitive_token $file]
   switch -exact [lindex $token 0] {
      beginproc {
         set token {procedure}
         while {"[lindex [set token2 [pstokens::get_token_rek $file]] 0]"\
               != "endproc" &amp;&amp; "[lindex $token2 0]" != "eof"} {
            lappend token $token2
         }
         if {"[lindex $token2 0]" == "eof"} then {
            if {[llength $token2]&gt;1} then {
               lappend token [lindex $token2 1]
            }
            set token [list eof $token]
         }
      }
      beginstring {set token [pstokens::get_string $file]}
      beginhex {set token [pstokens::get_hexstring $file]}
      beginbase85 {set token [pstokens::get_base85string $file]}
   }
   return $token
}
%   \end{tcl}
%   
%   These procedures, and in particular |get_token_rek|, could be made 
%   more intelligent when it comes to handling errors that occur inside 
%   procedures. Currently they just embed the |error| token in the 
%   procedure.
% \end{proc}\end{proc}
% 
% 
% \subsection{Writing tokens}
% 
% Writing tokens is simpler than reading them; the only real 
% complication is that the length of lines must be bounded. In order to 
% achieve this, data is written linewise to the file (much like the way 
% it is read).
% 
% \begin{arrayvar}{output_line}[file-id]
%   The |output_line| array stores each output line until it is 
%   actually written to file. Like with the |input_line| array, each 
%   active output file has its own entry in this array.
% \end{arrayvar}
% 
% \begin{proc}{flush_line}
%   The |flush_line| procedure has the syntax
%   \begin{quote}
%     |pstokens::flush_line| \word{file-id} \word{physical}\regopt
%   \end{quote}
%   It flushes buffered data that is to be written to the file with id 
%   \word{file-id}. The \word{physical} argument controls whether the 
%   data actually should be physically written to file (when |1|) or 
%   merely flushed from the |output_line| array (when |0|). The default 
%   is |0|. Calling |flush_line| starts a new line in the output file 
%   unless the |output_line| entry is empty.
%   \begin{tcl}
proc pstokens::flush_line {fileid {physical 0}} {
   global pstokens::output_line
   if {[string length [set pstokens::output_line($fileid)]]&gt;0} then {
      puts $fileid [set pstokens::output_line($fileid)]
      set pstokens::output_line($fileid) ""
   }
   if {$physical} then {flush $fileid}
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{open_output}
% \begin{proc}{close_output}
%   These procedures are wrappers around \Tcllogo's |open| and |close| 
%   procedures respectively which additionally initialize and unset the 
%   relevant entry of the |output_line|. Thus their syntaxes are
%   \begin{quote}
%     |pstokens::open_output| \word{file-name} \word{access}\regopt\ 
%     \word{permissions}\regopt\\
%     |pstokens::close_output| \word{file-id}
%   \end{quote}
%   and they return the same things as |open| and |close| respectively. 
%   Note however that unlike the case with |open|, the \word{access} 
%   argument of |open_output| defaults to |w|.
%   
%   A possible development of the routines for writing tokens would be 
%   to support output channels other than files. In that case more 
%   support data structures would have to be initialized, but then the 
%   |open_output| and |close_output| procedures would be redefined to 
%   cope with this.
%   \begin{tcl}
proc pstokens::open_output {name {access w} args} {
   set res [eval open \$name \$access $args]
   global pstokens::output_line
   set pstokens::output_line($res) ""
   return $res
}
% \medskip
proc pstokens::close_output {file_id} {
   pstokens::flush_line $file_id 1
   close $file_id
   global pstokens::output_line
   unset pstokens::output_line($file_id)
}
%   \end{tcl}
% \end{proc}\end{proc}
% 
% \begin{variable}{wrap_length}
%   The |wrap_length| variable stores the maximal number of characters 
%   that may appear on a single output line. The same value applies for 
%   all output files.
%   \begin{tcl}
set pstokens::wrap_length 72
%   \end{tcl}
% \end{variable}
% 
% \begin{proc}{put_word}
%   The |put_word| procedure has the syntax
%   \begin{quote}
%     |pstokens::put_word| \word{file-id} \word{string}
%   \end{quote}
%   It writes \meta{string} to the file with id \word{file-id}, 
%   preceeded by a whitespace character if the current line is 
%   nonempty. The whitespace character is usually a space, but it 
%   will be a newline if the length of what has already been written 
%   to the current line is too large to also fit the \word{string} 
%   within the length specified by the |wrap_length| variable.
%   \begin{tcl}
proc pstokens::put_word {file str} {
   global pstokens::output_line pstokens::wrap_length
   set l [string length [set pstokens::output_line($file)]]
   if {$l==0} then {
      set pstokens::output_line($file) $str
   } elseif {$l + [string length $str] &lt; ${pstokens::wrap_length}}\
   then {
      append pstokens::output_line($file) " $str"
   } else {
      puts $file [set pstokens::output_line($file)]
      set pstokens::output_line($file) $str
   }
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{put_breakable}
%   The |put_breakable| procedure has the syntax
%   \begin{quote}
%     |pstokens::put_breakable| \word{file-id} 
%     \word{prefix} \word{text} \word{suffix}
%   \end{quote}
%   It writes \meta{prefix}\meta{text}\meta{suffix} to the file with 
%   id \word{file-id} and inserts whitspace (newlines) into the 
%   \meta{text} in such a way that the |wrap_length| linewidth isn't 
%   exceeded. This is primarily useful for hexadecimal and base-85 
%   strings.
%   \begin{tcl}
proc pstokens::put_breakable {file prefix text suffix} {
   upvar #0 pstokens::output_line line pstokens::wrap_length wrap
   if {[string length $line($file)] + [string length $prefix] +\
      [string length $text] + [string length $suffix] &lt; $wrap} then {
      append line($file) " $prefix$text$suffix"
   } else {
      pstokens::flush_line $file
      set line($file) $prefix
      while {[string length $line($file)] + [string length $text] +\
            [string length $suffix] &lt; $wrap} {
         set t [expr {$wrap -[string length $line($file)]}]
         append line($file) [string range $text 0 [expr {$t-1}]]
         set text [string range $text $t end]
         pstokens::flush_line $file
      }
      append line($file) $text$suffix
   }
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{put_string}
%   The |put_string| procedure has the syntax
%   \begin{quote}
%     |pstokens::put_string| \word{file-id} \word{text}
%   \end{quote}
%   It writes |(|\meta{text}|)| to the file with id \word{file-id} and 
%   inserts backslash+newline items into the \meta{text} in such a way 
%   that the |wrap_length| linewidth isn't exceeded and the 
%   tokenization of the string is unaffected.
%   
%   The only places where it is unsafe to insert a backslash+newline is 
%   inside an escape squence. A position can only be inside such a 
%   sequence if the part of the \meta{text} which is before it gets a 
%   match against the regular expression
%   \begin{quote}
%     \verb"(^|[^\])(\\\\)*(\\[0-7]?[0-7]?)$"
%   \end{quote}
%   and in this case the position between the second and third 
%   parenthesis is safe.
%   \begin{tcl}
proc pstokens::put_string {file text} {
   upvar #0 pstokens::output_line line pstokens::wrap_length wrap
   if {[string length $line($file)] + [string length $text] + 2 &lt;\
         $wrap} then {
      append line($file) " ($text)"
   } else {
      pstokens::flush_line $file
      set line($file) (
      while {[string length $line($file)] + [string length $text] +\
            1 &lt; $wrap} {
         set t [expr {$wrap -[string length $line($file)]}]
         set s [string range $text 0 [expr {$t-1}]]
         if {[regexp {(^|[^\])(\\\\)*(\\[0-7]?[0-7]?)$} $s\
            foo bar baz escape]}\
         then {
            incr t -[string length $escape]
            set s [string range $text [expr {$t-1}]]
         }
         append line($file) $s
         set text [string range $text $t end]
         pstokens::flush_line $file
      }
      append line($file) $text)
   }
}
%   \end{tcl}
% \end{proc}
% 
% \begin{proc}{put_token}
%   The |put_token| procedure has the syntax
%   \begin{quote}
%     |pstokens::put_token| \word{file-id} \word{token}
%   \end{quote}
%   It writes the \PS\ token \word{token} to the file with id 
%   \word{file-id}.
%   \begin{tcl}
proc pstokens::put_token {file token} {
   switch [lindex $token 0] {
      number -
      radixnumber -
      executable {pstokens::put_word $file [lindex $token 1]}
      literal {pstokens::put_word $file /[lindex $token 1]}
      immediate {pstokens::put_word $file //[lindex $token 1]}
      string {pstokens::put_string $file [lindex $token 1]}
      comment {
         pstokens::flush_line $file 0
         puts $file %[lindex $token 1][lindex $token 2]
      }
      special {pstoken::put_token $file [lindex $token 1]}
      procedure {
         pstokens::put_word $file \{
         foreach subtok [lrange $token 1 end] {
            pstokens::put_token $file $subtok
         }
         pstokens::put_word $file \}
      }
      hexstring {pstokens::put_breakable $file &lt; [lindex $token 1] &gt;}
      base-85 {pstokens::put_breakable $file &lt;~ [lindex $token 1] ~&gt;}
      eof {
         if {[llength $token]&gt;1} then {
            pstokens::put_token $file [lindex $token 1]
         }
         pstokens::terminal_print "Token unexpectedly terminated by\
            end of file."
      }
      error {
         pstokens::terminal_print "Error token intercepted at put:\
            [switch [lindex $token 1] {
               extra {expr {"extra [lindex $token 2] at input line\
                  [lindex $token 3]"}}
               missing {expr {"missing [lindex $token 2] at input\
                  line [lindex $token 3]"}}
               missing/extra {expr {"missing [lindex $token 2], or\
                  extra [lindex $token 3], at input line\
                  [lindex $token 4]"}}
               default {expr {"!!!-unimplemented error-!!!"}}
            }]."
      }
      default {
         pstokens::terminal_print\
            "Token of unknown type encountered: $token"
      }
   }
}
%   \end{tcl}
% \end{proc}


% \begin{tcl}
%&lt;/tokenio&gt;
% \end{tcl}
% 
% \section{Token stream editing}
% 
% This section I still have ToDo.
% 
% 
% \begin{thebibliography}{99}
% 
% \bibitem{PSman}
%   Adobe Systems Incorporated: 
%   \textit{\PS\ language reference manual, 3rd ed.}, 
%   Addison--Wesley, 1999; ISBN 0-201-37922-8;
%   http:/\slash\texttt{partners.adobe.com}\slash
%   \texttt{asn}\slash\texttt{developer}\slash\texttt{PDFS}\slash
%   \texttt{TN}\slash\texttt{PLRM.pdf}.
% 
% \bibitem{MPinTUG}
%   John D.\ Hobby: \textit{A \MF-like System with \PS\ output}, 
%   TUGboat \textbf{10} (4) (1989), 505--512.
% %^^A 2 or 4 ?  Hobby's bibliography says 2, but Bebee's
% %^^A tugboat.bib says 4.
% %^^A @article{Hobby89a,
% %^^A    author = {John D. Hobby},
% %^^A    title = {A {METAFONT}-like System with PostScript Output},
% %^^A    journal = {{TUG}boat},
% %^^A    volume = {10},
% %^^A    number = {2},
% %^^A    pages = {505--512},
% %^^A    year = {1989}
% %^^A }
% \bibitem{MPman}
%   John D.\ Hobby: \textit{A User's Manual for \MP}, 
%   AT\&amp;T Bell Laboratories Computing Science Technical Report no.~162 
%   (1992);
%   \textit{see} http:/\slash \texttt{cm.bell-labs.com}\slash 
%   \texttt{who}\slash\texttt{hobby}\slash\texttt{MetaPost.html}.
% %^^A @techreport{Hobby92,
% %^^A    title = {A User's manual for {MetaPost}},
% %^^A    author = {John D. Hobby},
% %^^A    institution = {AT\&amp;T Bell Laboratories},
% %^^A    address = {Murray Hill, New Jersey},
% %^^A    type = {Computing Science Technical Report},
% %^^A    number = {no.~162},
% %^^A    year = {1992}
% %^^A }
% 
% \end{thebibliography}
% 
% 
% \Finale
% 
\endinput
</pre></body></html>