diff --git a/.ctan.sh b/.ctan.sh new file mode 100755 index 0000000..21c0821 --- /dev/null +++ b/.ctan.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +mkdir runcode + +cp -r runcode.sty runcode.tex runcode.pdf runcode_troubleshoot.tex runcode_troubleshoot.pdf README generated runcode/ + +tar cpvfhz runcode.tar.gz runcode/ + +rm -r runcode + +# tar cpvfhz runcode.tar.gz runcode.sty runcode.tex runcode.pdf troubleshoot.tex troubleshoot.pdf generated ./CTAN/README --transform='flags=r;s|./CTAN/README|README|' + + diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..10d754a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +.gitattributes export-ignore +.github export-ignore +.gitignore export-ignore +LICENSE export-ignore +FAQ.md export-ignore +paper export-ignore +examples export-ignore \ No newline at end of file diff --git a/.github/workflows/ctan.yml b/.github/workflows/ctan.yml new file mode 100644 index 0000000..953f4ba --- /dev/null +++ b/.github/workflows/ctan.yml @@ -0,0 +1,35 @@ +name: Release for CTAN + +on: + push: + tags: + - "v*.*" +jobs: + create_tarball: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 1 + - name: seupEmacs + uses: purcell/setup-emacs@master + with: + version: 28.1 + + - name: generate runcode.tex + run: ./.org2tex.sh + + - name: Compile LaTeX document + uses: xu-cheng/latex-action@v2 + with: + root_file: runcode.tex + latexmk_shell_escape: true + + - name: create CTAN tarball + run: ./.ctan.sh + + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + runcode.tar.gz diff --git a/.gitignore b/.gitignore index 8060280..39c3595 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ *.out *.synctex.gz *.mw +*generated *tmp Rplots.pdf @@ -22,8 +23,6 @@ __pycache__ *.pyc *debug.txt -.gitignore - *R.config *Rdebug.txt *julia.config @@ -31,5 +30,18 @@ __pycache__ *matlab.config *matlabdebug.txt serverPID*.txt +serverslist.txt +_minted-runcode/ FAQ.html +README +header.tex +runcode.tar.gz +runcode.tex +runcode.hd +runcode.pdf +runcode.tex~ +runcode/ + +# Exceptions +!examples/Collaborate/generated diff --git a/.org2tex.el b/.org2tex.el new file mode 100644 index 0000000..f7f70d5 --- /dev/null +++ b/.org2tex.el @@ -0,0 +1,69 @@ +(require 'package) +(package-initialize) +(unless package-archive-contents + (add-to-list 'package-archives '("org" . "https://orgmode.org/elpa/") t) + (add-to-list 'package-archives '("gnu" . "https://elpa.gnu.org/packages/") t) + (add-to-list 'package-archives '("melpa" . "https://melpa.org/packages/") t) + (package-refresh-contents)) +(dolist (pkg '(org)) + (unless (package-installed-p pkg) + (package-install pkg))) + +(require 'org) +;; (require 'oc-csl) +(require 'ox-publish) +;; (require 'projectile) + +(require 'ox-latex) +(add-to-list 'org-latex-classes + '("ltxdoc" + "\\documentclass{ltxdoc}" + ("\\section{%s}" . "\\section*{%s}") + ("\\subsection{%s}" . "\\subsection*{%s}") + ("\\subsubsection{%s}" . "\\subsubsection*{%s}") + ("\\paragraph{%s}" . "\\paragraph*{%s}") + ("\\subparagraph{%s}" . "\\subparagraph*{%s}"))) + +(setq + org-latex-hyperref-template "" + ;; org-confirm-babel-evaluate nil + org-hide-emphasis-markers t + org-latex-listings 'minted + org-latex-packages-alist '(("" "minted")) + ) + +;; (defun publish-html-and-patch (plist filename pub-dir) +;; "Export a html file then patch it by reversing lines" +;; (let ((outfile (org-html-publish-to-html plist filename pub-dir))) +;; (shell-command +;; (format "sed -i 's/Wang, H\\./Wang, H.<\\/strong>/' %s" +;; outfile (file-name-sans-extension outfile))))) + +;; (defvar OS--publish-project-alist +;; (list +;; (list "myweb" +;; :base-directory "./" +;; :exclude (regexp-opt '("others" "style/others")) +;; :base-extension "org" +;; :recursive t +;; :publishing-directory "./public" +;; ;; :publishing-function 'org-html-publish-to-html +;; :publishing-function 'publish-html-and-patch) +;; (list "attachments" +;; :base-directory "./" +;; :exclude (regexp-opt '("public" "others" "style/others")) +;; ;; :include '("CNAME" "keybase.txt" "LICENSE" ".nojekyll" "publish.el") +;; :recursive t +;; :base-extension (regexp-opt '("jpg" "gif" "png" "svg" "css" "pdf" "html")) +;; :publishing-directory "./public" +;; :publishing-function 'org-publish-attachment) +;; )) + +;; (defun OS-publish-all () +;; (interactive) +;; (let ((make-backup-files nil) +;; (org-publish-project-alist OS--publish-project-alist) +;; (org-html-htmlize-output-type 'css) +;; (org-cite-csl-styles-dir (expand-file-name "style/" (projectile-project-root)))) +;; (org-publish-all) +;; )) diff --git a/.org2tex.sh b/.org2tex.sh new file mode 100755 index 0000000..1b8d935 --- /dev/null +++ b/.org2tex.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +emacs README.org --batch --no-init-file --load .org2tex.el -f org-latex-export-to-latex --kill -f toggle-debug-on-error + +rl=`sed -n '7p' runcode.sty` +rl=`echo $rl | sed 's/.*\[\(.*\)\]/\1/'` +rl=`echo $rl | sed 's/runcode //'` +VersionDate=$(echo $rl | sed 's/\//\\\//g') + +sed "s/XXX-Date Version-XXX/$VersionDate/" CTAN/README > README +sed "s/XXX-Date Version-XXX/$VersionDate/" CTAN/header.tex > header.tex +sed -i 's/% Intended LaTeX compiler: pdflatex//' runcode.tex + +addheader=`cat header.tex ; cat runcode.tex` +echo "$addheader" > runcode.tex + + +# sed -i '1d' ./CTAN/README +# sed '1,1 s/.*/$VersionDate/' ./CTAN/README +# sed -re "s/XXX-Date Version-XXX/$(VersionDate)/" CTAN/README + +# # pdflatex -shell-escape runcode.tex diff --git a/CTAN/runcode/README b/CTAN/README similarity index 70% rename from CTAN/runcode/README rename to CTAN/README index 8df8d23..8e55e06 100644 --- a/CTAN/runcode/README +++ b/CTAN/README @@ -1,18 +1,21 @@ -LaTeX Package: runcode 2022/07/13 v1.4 +LaTeX Package: runcode XXX-Date Version-XXX ---------------------------------------- The runcode package enables the execution of source code (e.g., R, Julia, Matlab, shell, Python, etc.) and embed the results in the pdf file when compiling the LaTeX file. To use this package the shell-escape option must be enabled. -Copyright (C) 2020-2022 +Copyright (C) 2020-2025 Haim Bar and HaiYing Wang https://github.com/Ossifragus/runcode Files: - * README - this file - * runcode.sty - package source - * runcode.tex - documentation (source) - * runcode.pdf - documentation (pdf format) + * README - this file + * runcode.sty - package source + * runcode.tex - documentation (source) + * runcode.pdf - documentation (pdf format) + * runcode_troubleshoot.tex - documentation (source) + * runcode_troubleshoot.pdf - documentation (pdf format) + * generated/ - a folder which contains files generated by runcode for the runcode_troubleshoot file Installation: [manual installation] diff --git a/CTAN/header.tex b/CTAN/header.tex new file mode 100644 index 0000000..935273e --- /dev/null +++ b/CTAN/header.tex @@ -0,0 +1,13 @@ +% LaTeX Package: runcode XXX-Date Version-XXX +% +% Copyright (C) 2020-2025 by Haim Bar and HaiYing Wang +% +% This file may be distributed and/or modified under the +% conditions of the LaTeX Project Public License, either +% version 1.3c of this license or (at your option) any later +% version. The latest version of this license is in: +% +% http://www.latex-project.org/lppl.txt +% +% and version 1.3c or later is part of all distributions of +% LaTeX version 2005/12/01 or later. diff --git a/CTAN/runcode/runcode.pdf b/CTAN/runcode/runcode.pdf deleted file mode 100644 index a13ff9b..0000000 Binary files a/CTAN/runcode/runcode.pdf and /dev/null differ diff --git a/CTAN/runcode/runcode.sty b/CTAN/runcode/runcode.sty deleted file mode 120000 index 20e1f25..0000000 --- a/CTAN/runcode/runcode.sty +++ /dev/null @@ -1 +0,0 @@ -../../runcode.sty \ No newline at end of file diff --git a/CTAN/runcode/runcode.tex b/CTAN/runcode/runcode.tex deleted file mode 100644 index cb9edfe..0000000 --- a/CTAN/runcode/runcode.tex +++ /dev/null @@ -1,179 +0,0 @@ - % LaTeX Package: runcode v1.1 2021/04/17 - % - % Copyright (C) 2021 by Haim Bar and HaiYing Wang - % - % This file may be distributed and/or modified under the - % conditions of the LaTeX Project Public License, either - % version 1.3c of this license or (at your option) any later - % version. The latest version of this license is in: - % - % http://www.latex-project.org/lppl.txt - % - % and version 1.3c or later is part of all distributions of - % LaTeX version 2005/12/01 or later. - -\documentclass{ltxdoc} -\usepackage[hyphens]{url} -%\usepackage{runcode} -\ProvidesPackage{runcode}[2022/07/18 runcode v1.4] -\begin{document} - \title{The \textsf{runcode} package\thanks{This document corresponds to \textsf{runcode}~v1.4, dated~2022/07/18.}} - \author{Haim Bar and HaiYing Wang \\ \texttt{haim.bar@uconn.edu}, \texttt{haiying.wang@uconn.edu}} - \maketitle - -\abstract{\textsf{runcode} is a \LaTeX package that executes programming source codes (including all command line tools) from \LaTeX, and embeds the results in the resulting pdf file. Many programming languages can be easily used and any command-line executable can be invoked when preparing the pdf file from a tex file. - -It is recommended to use this package in the server-mode together with the Python talk2stat package. Currently, the server-mode supports Julia, MatLab, Python, and R. More languages will be added. - -For more details and usage examples and troubleshooting, refer to the package's github repository, at -\url{https://github.com/Ossifragus/runcode}. -} - -\section{Installation} -The package on CTAN can be installed automatically by your \TeX\space software (e.g., MikTeX Update Wizard). You can also simply put the runcode.sty file in the \LaTeX\space project folder. To use the package you have to enable the `shell-escape' option when compiling a \LaTeX\space document. - -The server mode requires the \textit{talk2stat} Python package. To install it from the command line, use: -\verb|pip3 install talk2stat|\\ -The \textit{talk2stat} source is available from \url{https://pypi.org/project/talk2stat/}. -Note that Python version 3.8.* and up is required. - -\section{Usage} -\subsection{Load the package} -\verb|\usepackage[options]{runcode}|\\ -\\ -Available options are: -\begin{itemize} -\item \texttt{julia}: start a \textit{talk2stat} server* for Julia [\url{https://julialang.org/}]. -\item \texttt{matlab}: start a \textit{talk2stat} server* for MatLab [\url{https://www.mathworks.com/products/matlab.html}]. -\item \texttt{R}: start a \textit{talk2stat} server* for R [\url{https://www.r-project.org/}]. -\item \texttt{run}: run source code, and store results in cache files. -\item \texttt{cache}: use cached results. -\item \texttt{stopserver}: stop the \textit{talk2stat} server(s) when the pdf compilation is done. -\item \texttt{nohup}: when using the server-mode, some editors terminate all child processes after LaTeX compiling such as Emacs with Auctex. For this case, use the nohup option. It set the variable notnohup to be false, and the server will not be terminated by the parent process. -\item \texttt{nominted}: use the \textit{fvextra} package [\url{https://ctan.org/pkg/fvextra}] instead of the \textit{minted} package [\url{https://ctan.org/pkg/minted}] to show code (\textit{fvextra} does not require Python's pygments package [\url{https://pygments.org/}], but it does not provide syntax highlights). -\end{itemize} -* Requires the Python package \textit{talk2stat} to be installed. - -\subsection{Basic commands} -\verb|\runExtCode{Arg1}{Arg2}{Arg3}[Arg4]| runs an external code. The arguments are: -\begin{itemize} -\item \texttt{Arg1} is the executable program. -\item \texttt{Arg2} is the source file name. -\item \texttt{Arg3} is the output file name (with an empty value, the counter `codeOutput` is used). -\item \texttt{Arg4} controls whether to run the code. \texttt{Arg4} is optional with three possible values: if skipped or with empty value, the value of the global Boolean variable \texttt{runcode} as determined by the \texttt{run} option when loading the package, is used; if the value is set to `run`, the code will be executed; if set to `cache` (or anything else), use cached results (see more about the cache below). -\end{itemize} - -\bigskip -\noindent \verb|\showCode{Arg1}{Arg2}[Arg3][Arg4]| shows the source code, using minted for a pretty layout or fvextra (if pygments is not installed). -\begin{itemize} -\item \texttt{Arg1} is the programming language. -\item \texttt{Arg2} is the source file name. -\item \texttt{Arg3} is the first line to show (optional with a default value 1). -\item \texttt{Arg4} is the last line to show (optional with a default value of the last line). -\end{itemize} - -\bigskip -\noindent \verb|\includeOutput{Arg1}[Arg2]| is used to embed the output from executed code. -\begin{itemize} -\item \texttt{Arg1} is the output file name, and it needs to have the same value as that of \texttt{Arg3} in \verb|\runExtCode|. If an empty value is given to \texttt{Arg1}, the counter `codeOutput` is used. -\item \texttt{Arg2} is optional and it controls the type of output with a default value `vbox` -\begin{itemize} -\item \texttt{vbox} (or skipped) = verbatim in a box. -\item \texttt{tex} = pure latex. -\item \texttt{inline} = embed result in text. -\end{itemize} -\end{itemize} - -\bigskip -\noindent \verb|\inln{Arg1}{Arg2}[Arg3]| is designed for simple calculations; it runs one command (or a short batch) and displays the output within the text. -\begin{itemize} -\item \texttt{Arg1} is the executable program or programming language. -\item \texttt{Arg2} is the source code. -\item \texttt{Arg3} is the output type. -\begin{itemize} -\item \texttt{inline} (or skipped or with empty value) = embed result in text. -\item \texttt{vbox} = verbatim in a box. -\end{itemize} -\end{itemize} - -\subsection{Language specific shortcuts} -\verb|\runJulia[Arg1]{Arg2}{Arg3}[Arg4]| runs an external Julia code file. -\begin{itemize} -\item \texttt{Arg1} is optional and uses \textit{talk2stat}'s Julia server by default. -\item \texttt{Arg2}, \texttt{Arg3}, and \texttt{Arg4} have the same effects as those of the basic command \verb|\runExtCode|. -\end{itemize} - -\noindent \verb|\inlnJulia[Arg1]{Arg2}[Arg3]| runs Julia source code (\texttt{Arg2}) and displays the output in line. -\begin{itemize} -\item \texttt{Arg1} is optional and uses the Julia server by default. -\item \texttt{Arg2} is the Julia source code to run. If the Julia source code is wrapped between \verb|```| on both sides (as in the markdown grammar), then it will be implemented directly; otherwise the code will be written to a file on the disk and then be called. -\item \texttt{Arg3} has the same effect as that of the basic command \verb|\inln|. -\end{itemize} - -\bigskip -\noindent \verb|\runMatLab[Arg1]{Arg2}{Arg3}[Arg4]| runs an external MatLab code file. -\begin{itemize} -\item \texttt{Arg1} is optional and uses \textit{talk2stat}'s MatLab server by default. -\item \texttt{Arg2}, \texttt{Arg3}, and \texttt{Arg4} have the same effects as those of the basic command \verb|\runExtCode|. -\end{itemize} - -\noindent -\verb|\inlnMatLab[Arg1]{Arg2}[Arg3]| runs MatLab source code (\texttt{Arg2}) and displays the output in line. -\begin{itemize} -\item \texttt{Arg1} is optional and uses the MatLab server by default. -\item \texttt{Arg2} is the MatLab source code to run. If the MatLab source code is wrapped between \verb|```| on both sides (as in the markdown grammar), then it will be implemented directly; otherwise the code will be written to a file on the disk and then be called. -\item \texttt{Arg3} has the same effect as that of the basic command \verb|\inln|. -\end{itemize} - - -\bigskip -\noindent \verb|\runR[Arg1]{Arg2}{Arg3}[Arg4]| runs an external R code file. -\begin{itemize} -\item \texttt{Arg1} is optional and uses \textit{talk2stat]}'s R server by default. -\item \texttt{Arg2}, \texttt{Arg3}, and \texttt{Arg4} have the same effects as those of the basic command \verb|\runExtCode|. -\end{itemize} - -\noindent \verb|\inlnR[Arg1]{Arg2}[Arg3]| runs R source code (\texttt{Arg2}) and displays the output in line. -\begin{itemize} -\item \texttt{Arg1} is optional and uses the R server by default. -\item \texttt{Arg2} is the R source code to run. If the R source code is wrapped between \verb|```| on both sides (as in the markdown grammar), then it will be implemented directly; otherwise the code will be written to a file on the disk and then be called. -\item \texttt{Arg3} has the same effect as that of the basic command \verb|\inln|. -\end{itemize} - - -\bigskip -\noindent \verb|\runPython[Arg1]{Arg2}{Arg3}[Arg4]| runs an external Python code file. -\begin{itemize} -\item \texttt{Arg1} is optional and uses \textit{talk2stat]}'s Python server by default. -\item \texttt{Arg2}, \texttt{Arg3}, and \texttt{Arg4} have the same effects as those of the basic command \verb|\runExtCode|. -\end{itemize} - -\noindent \verb|\inlnPython[Arg1]{Arg2}[Arg3]| runs Python source code (\texttt{Arg2}) and displays the output in line. -\begin{itemize} -\item \texttt{Arg1} is optional and uses the Python server by default. -\item \texttt{Arg2} is the R source code to run. If the Python source code is wrapped between \verb|```| on both sides (as in the markdown grammar), then it will be implemented directly; otherwise the code will be written to a file on the disk and then be called. -\item \texttt{Arg3} has the same effect as that of the basic command \verb|\inln|. -\end{itemize} - -\noindent \verb|\runPythonBatch[Arg1][Arg2]| runs an external Python source code (\texttt{Arg1}) in batch mode (without a server running). Python (at least currently), unlike the other languages we use, does not have an option to save and restore a session, which means that once a Python session ends, the working environment (variable, functions) is deleted. In order to allow a batch-mode in Python, we implemented such capability. It requires the dill module (https://pypi.org/project/dill/) module, which has to be installed via `pip3 install dill`. -\begin{itemize} -\item \texttt{Arg1} is the Python source code to run. -\item \texttt{Arg2} is the output file name. -\end{itemize} - - - - -\section{Revisions} -\begin{itemize} -\item v1.4, July 18, 2022: Fixed a bug in the cache mode. -\item v1.3, May 14, 2022: Removed the hard-coded minted options. -\item v1.2, May 3, 2022: Added python options (server and batch). -\item v1.1, April 17, 2021: Added a nohup option; improved error handling (missing code files, zero bytes in output files.) -\end{itemize} - -% -\section{Contributing} -We welcome your contributions to this package by opening issues on GitHub and/or making a pull request. We also appreciate more example documents written using \textsf{runcode}. - -\end{document} diff --git a/README.md b/README.md deleted file mode 100644 index 1514a4d..0000000 --- a/README.md +++ /dev/null @@ -1,242 +0,0 @@ -# runcode - -`runcode` is a LaTeX package that executes programming source codes (including -all command line tools) from LaTeX, and embeds the results in the resulting pdf -file. Many programming languages can be easily used and any command-line -executable can be invoked when preparing the pdf file from a tex file. `runcode` -is also available on [CTAN](https://ctan.org/pkg/runcode). - -It is recommended to use this package in the server mode together with the -[Python](https://www.python.org/) -[talk2stat](https://pypi.org/project/talk2stat/) package. Currently, the server -mode supports [Julia](https://julialang.org/), -[MatLab](https://www.mathworks.com/products/matlab.html), -[Python](https://www.python.org/), and [R](https://www.r-project.org/). More -languages will be added. - -**Citing `runcode`:** *Haim Bar and HaiYing Wang (2021). Reproducible Science -with LaTeX, [https://jds-online.org/journal/JDS/article/103/info] J. data -sci. 2021; 19, no. 1, 111-125, DOI 10.6339/21-JDS998* - -## Installation - -You can simply put the runcode.sty file in the LaTeX project folder. - -The server mode requires the [talk2stat](https://pypi.org/project/talk2stat/) -package. To install it from the command line, use: - -``` -pip3 install talk2stat -``` - -**Note**: `runcode` requires to enable the `shell-escape` option when compiling -a LaTeX document. - - - - - - - - -## Usage - -### Load the package: - -```latex -\usepackage[options]{runcode} -``` - -Available options are: - -- `cache`: use cached results. - -- `julia`: start server for [Julia](https://julialang.org/) (requires - [talk2stat](https://pypi.org/project/talk2stat/)). - -- `matlab`: start server for - [MatLab](https://www.mathworks.com/products/matlab.html) (requires - [talk2stat](https://pypi.org/project/talk2stat/)). - -- `nominted`: use the [fvextra](https://ctan.org/pkg/fvextra) package instead of - the [minted](https://ctan.org/pkg/minted) package to show code (this does not - require the [pygments](https://pygments.org/) package, but it does not provide - syntax highlights). - -- `nohup`: use the `nohup` command when starting a server. When using the - server-mode, some editors terminate all child processes after LaTeX compiling - such as Emacs with Auctex. This option set the variable notnohup to be false, - and the server will not be terminated by the parent process. **This option - has to be declared before declaring any language**, e.g., `[nohup, R]` works - but `[R, nohup]` does not work. - -- `python`: start server for [Python](https://www.python.org/) (requires - [talk2stat](https://pypi.org/project/talk2stat/)). - -- `run`: run source code. - -- `R`: start server for [R](https://www.r-project.org/) (requires - [talk2stat](https://pypi.org/project/talk2stat/)). - -- `stopserver`: stop the server(s) when the pdf compilation is done. - -**Note**: If [minted](https://ctan.org/pkg/minted) is used, the style of the code block is controlled through the minted package, [e.g.:](https://github.com/Ossifragus/runcode/blob/master/examples/MontyHall/MontyHall.tex#L3-L4) -```latex -\setminted[julia]{linenos, frame=single, bgcolor=bg, breaklines=true} -\setminted[R]{linenos, frame=single, bgcolor=lightgray, breaklines=true} -``` -The outputs from executing codes are displayed in [tcolorbox](https://ctan.org/pkg/tcolorbox?lang=en), so the style can be customized with `\tcbset`, [e.g.:](https://github.com/Ossifragus/runcode/blob/master/examples/MontyHall/MontyHall.tex#L5) -```latex -\tcbset{breakable,colback=red!5!white,colframe=red!75!black} -``` - -### Basic commands: - -- `\runExtCode{Arg1}{Arg2}{Arg3}[Arg4]` runs an external code. - - - `Arg1` is the executable program. - - `Arg2` is the source file name. - - `Arg3` is the output file name (with an empty value, the counter - `codeOutput` is used). - - `Arg4` controls whether to run the code. `Arg4` is optional with three - possible values: if skipped or with empty value, the value of the global - Boolean variable `runcode` is used; if the value is set to `run`, the code - will be executed; if set to `cache` (or anything else), use cached results - (see more about the cache below). - -- `\showCode{Arg1}{Arg2}[Arg3][Arg4]` shows the source code, using - [minted](https://ctan.org/pkg/minted) for a pretty layout or - [fvextra](https://ctan.org/pkg/fvextra) (if [pygments](https://pygments.org/) - is not installed). - - - `Arg1` is the programming language. - - `Arg2` is the source file name. - - `Arg3` is the first line to show (optional with a default value 1). - - `Arg4` is the last line to show (optional with a default value of the last - line). - -- `\includeOutput{Arg1}[Arg2]` is used to embed the output from executed code. - - - `Arg1` is the output file name, and it needs to have the same value as that - of `Arg3` in `\runExtCode`. If an empty value is given to `Arg1`, the - counter `codeOutput` is used. - - `Arg2` is optional and it controls the type of output with a default value - `vbox` - - `vbox` (or skipped) = verbatim in a box. - - `tex` = pure latex. - - `inline` = embed result in text. - -- `\inln{Arg1}{Arg2}[Arg3]` is designed for simple calculations; it runs one - command (or a short batch) and displays the output within the text. - - - `Arg1` is the executable program or programming language. - - `Arg2` is the source code. - - `Arg3` is the output type. - - `inline` (or skipped or with empty value) = embed result in text. - - `vbox` = verbatim in a box. - -### Language specific shortcuts: - -[Julia](https://julialang.org/) - -- `\runJulia[Arg1]{Arg2}{Arg3}[Arg4]` runs an external - [Julia](https://julialang.org/) code file. - - `Arg1` is optional and uses - [talk2stat](https://pypi.org/project/talk2stat/)'s - [Julia](https://julialang.org/) server by default. - - `Arg2`, `Arg3`, and `Arg4` have the same effects as those of the basic - command `\runExtCode`. -- `\inlnJulia[Arg1]{Arg2}[Arg3]` runs [Julia](https://julialang.org/) source - code (`Arg2`) and displays the output in line. - - `Arg1` is optional and uses the [Julia](https://julialang.org/) server by - default. - - `Arg2` is the [Julia](https://julialang.org/) source code to run. If the - [Julia](https://julialang.org/) source code is wrapped between "```" on both - sides (as in the markdown grammar), then it will be implemented directly; - otherwise the code will be written to a file on the disk and then be called. - - `Arg3` has the same effect as that of the basic command `\inln`. - -[MatLab](https://www.mathworks.com/products/matlab.html) - -- `\runMatLab[Arg1]{Arg2}{Arg3}[Arg4]` runs an external - [MatLab](https://www.mathworks.com/products/matlab.html) code file. - - `Arg1` is optional and uses - [talk2stat](https://pypi.org/project/talk2stat/)'s - [MatLab](https://www.mathworks.com/products/matlab.html) server by default. - - `Arg2`, `Arg3`, and `Arg4` have the same effects as those of the basic - command `\runExtCode`. -- `\inlnMatLab[Arg1]{Arg2}[Arg3]` runs - [MatLab](https://www.mathworks.com/products/matlab.html) source code (`Arg2`) - and displays the output in line. - - `Arg1` is optional and uses the - [MatLab](https://www.mathworks.com/products/matlab.html) server by default. - - `Arg2` is the [MatLab](https://www.mathworks.com/products/matlab.html) - source code to run. If the - [MatLab](https://www.mathworks.com/products/matlab.html) source code is - wrapped between "```" on both sides (as in the markdown grammar), then it - will be implemented directly; otherwise the code will be written to a file - on the disk and then be called. - - `Arg3` has the same effect as that of the basic command `\inln`. - -[R](https://www.r-project.org/) - -- `\runR[Arg1]{Arg2}{Arg3}[Arg4]` runs an external - [R](https://www.r-project.org/) code file. - - `Arg1` is optional and uses - [talk2stat](https://pypi.org/project/talk2stat/)'s - [R](https://www.r-project.org/) server by default. - - `Arg2`, `Arg3`, and `Arg4` have the same effects as those of the basic - command `\runExtCode`. -- `\inlnR[Arg1]{Arg2}[Arg3]` runs [R](https://www.r-project.org/) source code - (`Arg2`) and displays the output in line. - - `Arg1` is optional and uses the [R](https://www.r-project.org/) server by - default. - - `Arg2` is the [R](https://www.r-project.org/) source code to run. If the - [R](https://www.r-project.org/) source code is wrapped between "```" on both - sides (as in the markdown grammar), then it will be implemented directly; - otherwise the code will be written to a file on the disk and then be called. - - `Arg3` has the same effect as that of the basic command `\inln`. - - -[Python](https://www.python.org/) - -- `\runPython[Arg1]{Arg2}{Arg3}[Arg4]` runs an external - [Python](https://www.python.org/) code file. - - `Arg1` is optional and uses - [talk2stat](https://pypi.org/project/talk2stat/)'s - [Julia](https://julialang.org/) server by default. - - `Arg2`, `Arg3`, and `Arg4` have the same effects as those of the basic - command `\runExtCode`. -- `\inlnPython[Arg1]{Arg2}[Arg3]` runs [Python](https://www.python.org/) source - code (`Arg2`) and displays the output in line. - - `Arg1` is optional and uses the [Python](https://www.python.org/) server by - default. - - `Arg2` is the [Julia](https://julialang.org/) source code to run. If the - [Python](https://www.python.org/) source code is wrapped between "```" on both - sides (as in the markdown grammar), then it will be implemented directly; - otherwise the code will be written to a file on the disk and then be called. - - `Arg3` has the same effect as that of the basic command `\inln`. - -- `\runPythonBatch[Arg1][Arg2]` runs an external - [Python](https://www.python.org/) code file in batch mode (without a server running). - Python (at least currently), unlike the other languages we use, does not have an option - to save and restore a session, which means that once a Python session ends, the - working environement (variable, functions) is deleted. In order to allow a batch-mode - in Python, we implemented such capability. It requires the - [dill](https://pypi.org/project/dill/) module, which has to be installed via - `pip3 install dill`. - - `Arg1` is the [Python](https://www.python.org/) source file name, - - `Arg2` is the output file name. - - - - - - - - -## Contributing - -We welcome your contributions to this package by opening issues on GitHub and/or -making a pull request. We also appreciate more example documents written using -`runcode`. diff --git a/README.org b/README.org new file mode 100644 index 0000000..a38ef6a --- /dev/null +++ b/README.org @@ -0,0 +1,316 @@ +# -*- org-latex-hyperref-template: ""; org-latex-prefer-user-labels: t -*- + +#+startup: content hideblocks +#+options: H:2 timestamp:nil tex:t toc:nil author:nil +#+EXPORT_FILE_NAME: runcode.tex + +#+LaTeX_CLASS: ltxdoc +#+latex_header: \author{Haim Bar and HaiYing Wang \\haim.bar@uconn.edu, haiying.wang@uconn.edu} + +#+TITLE: The *runcode* package + +#+begin_export latex +\begin{abstract} +#+end_export + +=runcode= is a LaTeX package that executes programming source codes (including +all command line tools) from LaTeX, and embeds the results in the resulting pdf +file. Many programming languages can be easily used and any command-line +executable can be invoked when preparing the pdf file from a tex file. =runcode= +is also available on [[https://ctan.org/pkg/runcode][CTAN]]. + +It is recommended to use this package in the server mode together with the +[[https://www.python.org/][Python]] [[https://pypi.org/project/talk2stat/][talk2stat]] package. Currently, the server mode supports [[https://julialang.org/][Julia]], [[https://www.mathworks.com/products/matlab.html][MatLab]], +[[https://www.python.org/][Python]], and [[https://www.r-project.org/][R]]. More languages will be added. + +#+begin_export latex +For more details and usage examples and troubleshooting, refer to the +package’s github repository, at \url{https://github.com/Ossifragus/runcode}. + +\end{abstract} +#+end_export + +** Installation +:PROPERTIES: +:CUSTOM_ID: installation +:END: +You can simply put the runcode.sty file in the LaTeX project folder. + +The server mode requires the [[https://pypi.org/project/talk2stat/][talk2stat]] package, and some chunk command (e.g., +=\showChunk=) requires the [[https://pypi.org/project/advance-touch/][advance-touch]] package. To install them from the +command line, use: + +#+begin_example +pip3 install talk2stat advance-touch +#+end_example + +*Note*: =runcode= requires to enable the =shell-escape= option when +compiling a LaTeX document. + + +** Usage +:PROPERTIES: +:CUSTOM_ID: usage +:END: +*** Load the package: +:PROPERTIES: +:CUSTOM_ID: load-the-package +:END: +#+begin_src latex :exports code +\usepackage[options]{runcode} +#+end_src + +Available options are: + +- =cache=: use cached results. + +- =fvextra=: use the [[https://ctan.org/pkg/fvextra][fvextra]] package to show code. + +- =julia=: start server for [[https://julialang.org/][Julia]] (requires [[https://pypi.org/project/talk2stat/][talk2stat]]). + +- =listings=: use the [[https://ctan.org/pkg/listings?lang=en][listings]] package to show code. + +- =matlab=: start server for [[https://www.mathworks.com/products/matlab.html][MatLab]] (requires [[https://pypi.org/project/talk2stat/][talk2stat]]). + +- =minted=: use the [[https://ctan.org/pkg/minted][minted]] package to show code (requires the [[https://pygments.org/][pygments]] package). + This is the default option. + +- =nominted=: use the [[https://ctan.org/pkg/fvextra][fvextra]] package + instead of the [[https://ctan.org/pkg/minted][minted]] package to show + code (this does not require the [[https://pygments.org/][pygments]] + package, but it does not provide syntax highlights). + +- =nohup=: use the =nohup= command when starting a server. When using + the server-mode, some editors terminate all child processes after + LaTeX compiling such as Emacs with Auctex. This option set the + variable notnohup to be false, and the server will not be terminated + by the parent process. *This option has to be declared before + declaring any language*, e.g., =[nohup, R]= works but =[R, nohup]= + does not work. + +- =python=: start server for [[https://www.python.org/][Python]] + (requires [[https://pypi.org/project/talk2stat/][talk2stat]]). + +- =run=: run source code. + +- =R=: start server for [[https://www.r-project.org/][R]] (requires + [[https://pypi.org/project/talk2stat/][talk2stat]]). + +- =stopserver=: stop the server(s) when the pdf compilation is done. + +*Note*: If [[https://ctan.org/pkg/minted][minted]] is used, the style of +the code block is controlled through the minted package, +[[https://github.com/Ossifragus/runcode/blob/master/examples/MontyHall/MontyHall.tex#L3-L4][e.g.:]] + +#+begin_src latex :exports code +\setminted[julia]{linenos, frame=single, bgcolor=bg, breaklines=true} +\setminted[R]{linenos, frame=single, bgcolor=lightgray, breaklines=true} +#+end_src + +Similarly, [[https://ctan.org/pkg/fvextra][fvextra]] and [[https://ctan.org/pkg/listings?lang=en][listings]] packages can be customized through the =\fvset= +and =\lstset= commands, respectively, e.g.: + +#+begin_src latex :exports code +\fvset{fontsize=\small, linenos=true, frame=single} +\lstset{basicstyle=\large, frame=single} +#+end_src + +The outputs from executing codes are displayed in +[[https://ctan.org/pkg/tcolorbox?lang=en][tcolorbox]], so the style can +be customized with =\tcbset=, +[[https://github.com/Ossifragus/runcode/blob/master/examples/MontyHall/MontyHall.tex#L5][e.g.:]] + +#+begin_src latex :exports code +\tcbset{breakable,colback=red!5!white,colframe=red!75!black} +#+end_src + +*** Basic commands: +:PROPERTIES: +:CUSTOM_ID: basic-commands +:END: +- =\runExtCode{Arg1}{Arg2}{Arg3}[Arg4]= runs an external code. + + - =Arg1= is the executable program. + - =Arg2= is the source file name. + - =Arg3= is the output file name (with an empty value, the counter =codeOutput= is + used). + - =Arg4= controls whether to run the code. =Arg4= is optional with three possible + values. + - empty value or skipped: the value of the global Boolean variable =runcode= + is used; + - =run=: the code will be executed; + - =cache= (or anything else): use cached results; if the output file does not + exist, the cache option will be overridden and the code will run. + +- =\showCode{Arg1}{Arg2}[Arg3][Arg4]= shows the source code, using + [[https://ctan.org/pkg/minted][minted]] (requires [[https://pygments.org/][pygments]]), [[https://ctan.org/pkg/fvextra][fvextra]], or [[https://ctan.org/pkg/listings?lang=en][listings]]. + + - =Arg1= is the programming language. + - =Arg2= is the source file name. + - =Arg3= is the first line to show (optional with a default value 1). + - =Arg4= is the last line to show (optional with a default value of + the last line). + +- =\includeOutput{Arg1}[Arg2]= is used to embed the output from executed + code. + + - =Arg1= is the output file name, and it needs to have the same value + as that of =Arg3= in =\runExtCode=. If an empty value is given to + =Arg1=, the counter =codeOutput= is used. + - =Arg2= is optional and it controls the type of output with a default + value =vbox=. + - =vbox= (or skipped) = verbatim in a box; + - =tex= = pure latex; + - =inline= = embed result in text. + +- =\inln{Arg1}{Arg2}[Arg3][Arg4]= is designed for simple calculations; it runs + one command (or a short batch) and displays the output within the + text. + + - =Arg1= is the executable program or programming language. + - =Arg2= is the source code. + - =Arg3= is the output file name (optional); + - =Arg4= is the output type and controls whether to run the code. + - =inline= (or skipped or with empty value): embed result in text; + - =vbox=: verbatim in a box; + - =cache= or appending =.cache= to the argument (e.g., =vbox.cache=) AND the + output file in =Arg3= exists: the cached result will be used and the code + will not run. + +- =\showChunk{Arg1}{Arg2}[Arg3][Arg4]= prints a selected chunk from source + code. The chunk is identified in the source code by two strings that define + the beginning and end of the chunk. The default beginning is =label==== + where == should be a unique user-defined chunk ID. The default end marker is + ~===end~. In the code, these markers should appear after a comment character, so + that the code will run. + + - =Arg1= is the programming language. + - =Arg2= is the source file name. + - =Arg3= is is the chunk identifier. + - =Arg4= and =Arg5= are the beginning and end markings of a chunk (optional with + default =label=== and ~===end~. + + + +*** Extended commands: +:PROPERTIES: +:CUSTOM_ID: extended-commands +:END: + +- =\runCodeIncOut{Arg1}{Arg2}[Arg3][Arg4][Arg5]= runs an external code and + embeds the output. This is a combination of =\runExtCode= and =\includeOutput=. + - =Arg1= is the executable program. + - =Arg2= is the source file name. + - =Arg3= (optional) controls whether to run the code. Its functionality is the + same as that of =Arg4= of =\runExtCode=. + - =Arg4= (optional) is the output file name. Its functionality is the same as + that of =Arg3= of =\runExtCode=. + - =Arg5= (optional) controls the type of output with a default value =vbox=. Its + functionality is the same as that of =Arg3= of =\includeOutput=. + +*** Language specific shortcuts: +:PROPERTIES: +:CUSTOM_ID: language-specific-shortcuts +:END: + +Replace =LANG= with =Julia=, =MatLab=, =Python=, or =R=, for the [[https://julialang.org/][Julia]], [[https://www.mathworks.com/products/matlab.html][MatLab]], [[https://www.python.org/][Python]], [[https://www.r-project.org/][R]] +language, respectively, for the following commands. + +- =\runLANG[Arg1]{Arg2}{Arg3}[Arg4]= runs an external =LANG= code file. + - =Arg1= is optional and uses [[https://pypi.org/project/talk2stat/][talk2stat]]'s =LANG= server by default. + - =Arg2=, =Arg3=, and =Arg4= have the same effects as those of the basic command + =\runExtCode=. +- =\runLANGIncOut[Arg1]{Arg2}[Arg3][Arg4][Arg5]= runs an external =LANG= code + file and embeds the output. + - =Arg1= is optional and uses [[https://pypi.org/project/talk2stat/][talk2stat]]'s =LANG= server by default. + - =Arg2=, =Arg3=, =Arg4=, and =Arg5= have the same effects as those of the command + =\runCodeIncOut=. +- =\inlnLANG[Arg1]{Arg2}[Arg3][Arg4]= runs =LANG= source code (=Arg2=) and displays + the output in line. + - =Arg1= is optional and uses the =LANG= server by default. + - =Arg2= is the =LANG= source code to run. If the =LANG= source code is wrapped + between "=```=" on both sides (as in the markdown grammar), then it will be + implemented directly; otherwise the code will be written to a file on the + disk and then be called. + - =Arg3= and =Arg4= have the same effects as those of the basic command =\inln=. + +- =\runLANGChunk[Arg1]{Arg2}{Arg3}[Arg4][Arg5][Arg6]= runs a selected chunk from + an external =LANG= code file and embeds the output. + - =Arg1= (optional ) uses [[https://pypi.org/project/talk2stat/][talk2stat]]'s =LANG= server by default. + - =Arg2= is the source file name. + - =Arg3= is is the chunk identifier. + - =Arg4= (optional) controls whether to run the code. + - =Arg5= (optional) is the output file name. + - =Arg6= (optional) controls the type of output with a default value =vbox=. + +For example, +#+begin_src latex :exports code +\runR{code/MontyHall_1.R}{montyhall-R1} +\runRIncOut{code/MontyHall_1.R}[][montyhall-R1] +#+end_src + +Specifically for [[https://www.python.org/][Python]], + +- =\runPythonBatch[Arg1][Arg2]= runs an external [[https://www.python.org/][Python]] code file in batch mode + (without a server running). Python (at least currently), unlike the other + languages we use, does not have an option to save and restore a session, which + means that once a Python session ends, the working environment (variable, + functions) is deleted. In order to allow a batch-mode in Python, we + implemented such capability. It requires the [[https://pypi.org/project/dill/][dill]] module, which has to be + installed via =pip3 install dill=. + - =Arg1= is the [[https://www.python.org/][Python]] source file name, + - =Arg2= is the output file name. + + +** Revisions +- v2.4, January 17, 2025: 1. put file names within $$ for more robust + warnings; 2. use the python [[https://pypi.org/project/advance-touch/][advance-touch]] package to create sub-folders for + chunk related commands. +- v2.3, January 12, 2024: two bug fixes (1. removed an extra space after + ~\inlnX~; 2. when an underscore appeared inside R code, the tex file would + compile with an error). +- v2.2, September 8, 2023: add ~\showChunk~ basic command and ~\runLANGChunk~ + commands for multiple languages. +- v2.1, June 30, 2023: detokenize code which is passed to =\inln=. This is + necessary when the code contains special latex characters like backslash. + These characters are escaped by latex when they are passed to macros as arguments, + which, in the case of =\inln=, alters the correct syntax and causes execution errors. + Detokenizing ensures that the code is passed as-is to R/Julia/MatLab/Python. + We thank [[https://github.com/kiryph][kiryph]] for reporting the issue. +- v2.0, June 23, 2023: add =\runCodeIncOut= command that runs an external code and + embeds the output and add some language specific shortcuts; update + =\runExtCode= so that if the output file does not exist, the =cache= option will + be overridden and the code will run; update =\inln= so that its =Arg4= accept + =cache= or appending =.cache= to use the cached result. We thank [[https://github.com/kiryph][kiryph]] for + suggesting these features. +- v1.9, June 13, 2023: update =\inln= command; the optional =Arg3= is the output + file name and the optional =Arg4= is the output type. +- v1.8, January 18, 2023, add support to [[https://ctan.org/pkg/listings?lang=en][listings.]] +- v1.7, August 20, 2022: changed the tmp/ folder to generated/ in order to + conform with CTAN suggestions; renamed the troubleshooting file. +- v1.6, August 10, 2022: stop only configured/running servers; a new + reducedspace option - some document classes put more space after the code box; + changed the default timeout of servers to 60 seconds; expanded the + troubleshooting document. New examples are now available on GitHub, including + how to collaborate with people who use Overleaf. +- v1.5, July 23, 2022: Removed the utf8x option when loading inputenc due to a + conflict with hyperref. +- v1.4, July 18, 2022: Fixed a bug in the cache mode. +- v1.3, May 14, 2022: Removed the hard-coded minted options. +- v1.2, May 3, 2022: Added python options (server and batch). +- v1.1, April 17, 2021: Added a nohup option; improved error handling (missing + code files, zero bytes in output files.) + +** Contributing +:PROPERTIES: +:CUSTOM_ID: contributing +:END: +We welcome your contributions to this package by opening issues on +GitHub and/or making a pull request. We also appreciate more example +documents written using =runcode=. + + +*Citing* =runcode=: +Haim Bar and HaiYing Wang (2021). [[https://jds-online.org/journal/JDS/article/103/info][Reproducible Science with LaTeX]], +/Journal of Data Science/ 2021; 19, no. 1, 111-125, DOI 10.6339/21-JDS998 + diff --git a/examples/Collaborate/ForceCache b/examples/Collaborate/ForceCache new file mode 100644 index 0000000..e69de29 diff --git a/examples/Collaborate/Makefile b/examples/Collaborate/Makefile new file mode 100644 index 0000000..4cacad1 --- /dev/null +++ b/examples/Collaborate/Makefile @@ -0,0 +1,17 @@ +# Use the cache, compile from tmp files, remove temporary code files +overleaf: + touch ForceCache + xelatex -shell-escape runcodeOL.tex + +clean: + rm -f runcodeOL.synctex.gz runcodeOL.aux tmp/*.txt runcodeOL.mw runcodeOL.log + +# stop the talk2stat server, but don't compile the book: +stopserver: clean + python3 -c 'from talk2stat.talk2stat import client; client("./","python","QUIT")' + rm -f serverPIDpython.txt pythondebug.txt talk2stat.log nohup.out + +# build the book and use the server, not the cache option: +build: clean + rm -f ForceCache nohup.out + xelatex -shell-escape runcodeOL.tex diff --git a/examples/Collaborate/code/Levenshtein.py b/examples/Collaborate/code/Levenshtein.py new file mode 100644 index 0000000..08b3806 --- /dev/null +++ b/examples/Collaborate/code/Levenshtein.py @@ -0,0 +1,23 @@ +def Levenshtein(s1, s2): + n = len(s1) + m = len(s2) + d = [0] * (n+1) + for x in range (n+1): + d[x] = [0] * (m+1) + for i in range (n): + d[i+1][0] = i+1 + for j in range (m): + d[0][j+1] = j+1 + for i in range (n): + for j in range (m): + if s1[i] == s2[j]: + substitutionCost = 0 + else: + substitutionCost = 1 + d[i+1][j+1] = min(d[i][j+1] + 1, # deletion + d[i+1][j] + 1, # insertion + d[i][j] + substitutionCost) # substitution + return(d[n][m]) + +str1 = "sibling" +str2 = "setting" diff --git a/examples/Collaborate/generated/levdist.tex b/examples/Collaborate/generated/levdist.tex new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/Collaborate/generated/levdist.tex @@ -0,0 +1 @@ + diff --git a/examples/Collaborate/generated/runcodeOL_inln2.tex b/examples/Collaborate/generated/runcodeOL_inln2.tex new file mode 100644 index 0000000..e620d96 --- /dev/null +++ b/examples/Collaborate/generated/runcodeOL_inln2.tex @@ -0,0 +1 @@ +sibling diff --git a/examples/Collaborate/generated/runcodeOL_inln3.tex b/examples/Collaborate/generated/runcodeOL_inln3.tex new file mode 100644 index 0000000..09609a0 --- /dev/null +++ b/examples/Collaborate/generated/runcodeOL_inln3.tex @@ -0,0 +1 @@ +setting diff --git a/examples/Collaborate/generated/runcodeOL_inln4.tex b/examples/Collaborate/generated/runcodeOL_inln4.tex new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/examples/Collaborate/generated/runcodeOL_inln4.tex @@ -0,0 +1 @@ +3 diff --git a/examples/Collaborate/python.config b/examples/Collaborate/python.config new file mode 100644 index 0000000..4c8ecf5 --- /dev/null +++ b/examples/Collaborate/python.config @@ -0,0 +1,4 @@ +[SERVERCONFIG] +PORT = 65433 +DEBUGFILE = pythondebug.txt +PIPETIMEOUT = 300 diff --git a/examples/Collaborate/runcodeOL.pdf b/examples/Collaborate/runcodeOL.pdf new file mode 100644 index 0000000..15d22b6 Binary files /dev/null and b/examples/Collaborate/runcodeOL.pdf differ diff --git a/examples/Collaborate/runcodeOL.tex b/examples/Collaborate/runcodeOL.tex new file mode 100644 index 0000000..982c3e0 --- /dev/null +++ b/examples/Collaborate/runcodeOL.tex @@ -0,0 +1,108 @@ +%%% Local Variables: +%%% TeX-command-extra-options: "-shell-escape" +%%% End: + +\documentclass[12pt]{article} +\usepackage[a4paper, total={7in, 10in}]{geometry} + +\edef\TeXjobname{\jobname} % keep a copy (not essential) +\edef\jobname{\detokenize{runcodeOL}} + +\IfFileExists{ForceCache}{ +\usepackage[cache]{../../runcode} +} +{ +\usepackage[nohup,run,python,stopserver]{../../runcode} % can still add/remove the cache option here +} + +\tcbset{colback=blue!5!white,colframe=blue!75!black} + +\begin{document} +\title{Working with collaborators (including through Overleaf)} +\maketitle + +You may work with a group of co-authors and share your project's folder using GitHub, Dropbox, or a similar file sharing system. +It is possible that the work is divided in such a way that some collaborators work only on the text, and are not involved in, or just cannot run the code. For example, they may not have Python, R, Julia, or Matlab on their computer, or they may be subject-matter experts with no programming experience. +It is possible for them to change into cache mode every time they compile the document, but it is cumbersome, and since the files are shared it means that they change the run/cache setting for all other collaborators. In this document we describe how such a group can work efficiently so that the coders can compile in server-mode, and the non-coders will always compile in cache mode. + +We recently started working with a publisher and our collaborators there have a premium Overleaf account, which means that they can share files with us via GitHub. However, as of 2022, it is not possible for Overleaf users to install python packages, so our collaborators cannot install the talk2stat package, which is needed for runcode. This means that when they compile our book, they must do it in cache mode. To make it work, we added a few lines to the preamble and created a make file. Both are described in detail below. The steps below are specific to this example, but can be easily generalized to any similar collaborative project where files are shared by multiple authors. + +\subsection*{The preamble} + +First, in the preamble of runcodeOL.tex we put the following: +\begin{tcolorbox} +\begin{Verbatim} +\IfFileExists{ForceCache}{ +\usepackage[cache]{runcode} +} +{ +\usepackage[nohup,run,python,stopserver]{runcode} +} +\end{Verbatim} +\end{tcolorbox} + +If there is a file called ForceCache in the project's directory, the book is compiled in cache mode. Otherwise, we use the server-mode (using Python, in this example.) See more about this in the next section. + +We use the stopserver option, which is not essential, but we recommend doing it when the project is in its early stages and many changes are made to the code and to the configuration of the server, and the cache files might become outdated. + +The nohup option is only used because Emacs (for example) always stops all child processes (like talk2stat), whether we use stopserver or not. The nohup option prevents this from happening and then the choice whether to keep the server running or not, is up to us. + +Note that we used the run option here, but we can still manually change it to cache if needed, or override individual invocations of code within the document to be in cache mode. + +\bigskip +We also found that when Overleaf compiles the document, temporary files names do not get the main file's prefix. For example, the main file for this example is runcodeOL.tex so normally a temporary file created by \LaTeX~ will have runcodeOL as a prefix, but this is not the case with Overleaf. To force the prefix to be the main file's name, which is requires so that runcode can embed code and results in the pdf, we add the following lines to the preamble of runcodeOL.tex: +\begin{tcolorbox} +\begin{Verbatim} +\edef\TeXjobname{\jobname} % (not really essential) +\edef\jobname{\detokenize{runcodeOL}} +\end{Verbatim} +\end{tcolorbox} + +\subsection*{The make file} + +In order to allow us to compile the book and run the code, but allow our publisher to compile the \LaTeX~ file without having to run the code, we created a file called Makefile with four options: +\begin{itemize} + \item \textbf{build}: build the book and use the server-mode. + \item \textbf{overleaf}: use the cache, compile from generated files (this must be done after make build, or else the generated files will not be up to date). + \item \textbf{clean}: as the name suggests. + \item \textbf{stopserver}: runs clean, and then stops the talk2stat server. +\end{itemize} + + +The content of our Makefile is this: +\showCode{bash}{Makefile} + +Before pushing the code to GitHub, we run the following sequence of commands in the command line: +\begin{tcolorbox} +\begin{Verbatim} +make stopserver +make build +make overleaf +\end{Verbatim} +\end{tcolorbox} +If GitHub is used, we then commit the changes and push to the GitHub server. With Dropbox, OneDrive, etc., the updates will be available to our collaborators without extra steps. + +Here is a detailed explanation of the steps: +\begin{enumerate} + \item \verb|make stopserver| is used to first remove the .aux file and any code files created by runcode. It's possible to add to the clean option the following suffixes: .bbl, .idx, .mw, .tbc, .toc,, or the cache files, \verb|generated/*|\\ + Then, it stops the talk2stat server, and deletes the files associated with it. + \item \verb|make build| removes the file ForceCache in order to ensure that the next compilation will be in server-mode (no cache).\\ + Then, the project is compiled (with the mandatory \verb|shell-escape| option). + \item \verb|make overleaf| first creates the ForceCache file, so that the next compilation will not invoke the python code.\\ + Then, the code is compiled one more time. This is not essential -- our collaborators can do it when the pull the latest changes, but it's good to test that the cache-mode compilation worked properly (and save our collaborators the extra step). +\end{enumerate} + + +\subsection*{Example} +Try compiling this file with the aforementioned steps, and see that after the first compilation the files under the generated folder are updated, but after running \verb|make overleaf| only the main pdf file is updated. + +In this example we demonstrate execution of Python code (which computes the Levenshtein distance between two words.) + +\showCode{python}{code/Levenshtein.py} +\runPython{code/Levenshtein.py}{levdist} + +The Levenshtein distance between the words \inlnPython{```print(str1)```} and \inlnPython{```print(str2)```} is \inlnPython{```print(Levenshtein(str1, str2))```}. + +\end{document} + + diff --git a/examples/MontyHall/MontyHall.pdf b/examples/MontyHall/MontyHall.pdf new file mode 100644 index 0000000..d5d167d Binary files /dev/null and b/examples/MontyHall/MontyHall.pdf differ diff --git a/examples/MontyHall/MontyHall.tex b/examples/MontyHall/MontyHall.tex index cd544b5..ef86681 100644 --- a/examples/MontyHall/MontyHall.tex +++ b/examples/MontyHall/MontyHall.tex @@ -1,5 +1,5 @@ \documentclass[12pt]{article} -\usepackage[nohup,run,julia,R]{../../runcode} +\usepackage[nohup,run,julia,R,stopserver]{../../runcode} \setminted[julia]{fontsize=\normalsize, linenos, frame=single, bgcolor=bg} \setminted[R]{linenos, frame=single, bgcolor=lightgray, breaklines=true} \tcbset{breakable,colback=red!5!white,colframe=red!75!black} @@ -29,19 +29,19 @@ \NewDocumentCommand{\showR}{m O{} O{}}{} \renewcommand{\runR}[2]{} \newcommand{\includeR}[1]{} - \renewcommand{\inlnR}[1]{} + \RenewDocumentCommand{\inlnR}{O{}mO{}O{}}{} \fi -\newif\ifuseJulia\useJuliatrue +\newif\ifuseJulia\useJuliafalse \ifuseJulia \NewDocumentCommand{\showJulia}{m O{} O{}}{\showCode{julia}{#1}[#2][#3]} \NewDocumentCommand{\includeJulia}{m O{vbox}}{\includeOutput{#1}[#2]} - \renewcommand{\inlnR}[1]{} + \RenewDocumentCommand{\inlnR}{O{}mO{}O{}}{} \else \NewDocumentCommand{\showJulia}{m O{} O{}}{} \renewcommand{\runJulia}[2]{} \newcommand{\includeJulia}[1]{} - \renewcommand{\inlnJulia}[1]{} + \RenewDocumentCommand{\inlnJulia}{O{}mO{}O{}}{} \fi %%%% ------------------------------------------------ @@ -77,14 +77,14 @@ \runR{code/MontyHall_2.R}{montyhall-R2} % \includeOutput{montyhall-R2} -The approximate probabilities of keeping the original choice and switching are \inlnR{```cat(probabilities)```}\inlnJulia{```print(probabilities)```}. +The approximate probabilities of keeping the original choice and switching are \inlnR{```cat(probabilities)```}[inlnR1]\inlnJulia{```print(probabilities)```}[inlnJulia1]. \end{document} %%% Local Variables: %%% coding: utf-8 %%% TeX-command-extra-options: "--shell-escape" -%%% TeX-engine: luatex +%%% TeX-engine: xetex %%% End: -%%% TeX-engine: xetex \ No newline at end of file +%%% TeX-engine: luatex \ No newline at end of file diff --git a/examples/ObesityWithSidenotes/Code/obesityPlot1.R b/examples/ObesityWithSidenotes/Code/obesityPlot1.R index 835948a..b1be93e 100644 --- a/examples/ObesityWithSidenotes/Code/obesityPlot1.R +++ b/examples/ObesityWithSidenotes/Code/obesityPlot1.R @@ -1,4 +1,4 @@ -pdf("tmp/shareDelta.pdf",width = 4,height = 4) +pdf("generated/shareDelta.pdf",width = 4,height = 4) plot(widedata$"1975",widedata$"2016"-widedata$"1975",ylim=c(0,40),col=0,xlab="Share obese in 1975", ylab="change in share obese from 1975 to 2016") grid() @@ -6,4 +6,4 @@ usarow <- which(widedata$Code == "USA") text(widedata$"1975",widedata$"2016"-widedata$"1975",widedata$Code,cex=0.6,col="slateblue1") text(widedata$"1975"[usarow],widedata$"2016"[usarow]-widedata$"1975"[usarow],col=2,cex=0.6, widedata$Code[usarow]) -dev.off() \ No newline at end of file +dev.off() diff --git a/examples/ObesityWithSidenotes/ObesityWithSidenotes.pdf b/examples/ObesityWithSidenotes/ObesityWithSidenotes.pdf index b68e17c..70d7100 100644 Binary files a/examples/ObesityWithSidenotes/ObesityWithSidenotes.pdf and b/examples/ObesityWithSidenotes/ObesityWithSidenotes.pdf differ diff --git a/examples/ObesityWithSidenotes/ObesityWithSidenotes.tex b/examples/ObesityWithSidenotes/ObesityWithSidenotes.tex index d8f7edf..ef874fb 100644 --- a/examples/ObesityWithSidenotes/ObesityWithSidenotes.tex +++ b/examples/ObesityWithSidenotes/ObesityWithSidenotes.tex @@ -1,7 +1,7 @@ \documentclass{caesar_book} \usepackage{marginnote} -\usepackage[run]{runcode} +\usepackage[run]{../../runcode} %\usepackage[nominted]{runcode} \usepackage{hyperref} @@ -16,14 +16,14 @@ \chapter{Example -- Global Trends in Obesity} The data for this example has been downloaded from the web-site `Our World in Data' on Sept. 25, 2020. The dataset contains the share of obese people in each country from 1975 to 2016. Obesity is defined as having a body-mass index equal to or greater than 30.\marginnote{\url{https://ourworldindata.org}} -\begin{filecontents*}{tmp/fifteenfold.R} +\begin{filecontents*}{generated/fifteenfold.R} library(xtable) fifteenfold <- which(widedata$"2016" > 15*widedata$"1975") print(xtable(widedata[fifteenfold,c(3,44)], caption = "Countries in which the share of obese people increased more than 15-fold from 1975 to 2016", label="fifteenfold")) \end{filecontents*} -\runExtCode{Rscript --save --restore}{tmp/fifteenfold.R}{fifteenFold}{} +\runExtCode{Rscript --save --restore}{generated/fifteenfold.R}{fifteenFold}{} -In the United states, the share of obese people rose from \inln{Rscript --save --restore}{cat(as.numeric(widedata[which(widedata$Code == "USA"),3]))} percent in 1975, to \inln{Rscript --save --restore}{cat(as.numeric(widedata[which(widedata$Code == "USA"),44]))} in 2016. +In the United states, the share of obese people rose from \inln{Rscript --save --restore}{cat(as.numeric(widedata[which(widedata$Code == "USA"),3]))}[wide1] percent in 1975, to \inln{Rscript --save --restore}{cat(as.numeric(widedata[which(widedata$Code == "USA"),44]))}[wide2] in 2016. \marginnote{We used the inln command to generate this output.} In all countries the share of obese people has increased between 1975 and 2016, but 24 saw more than a ten-fold increase (many of which in South-East Asia, where the share of obese people grew from 0.4\% of the population to 4.7\%). \includeOutput{fifteenFold}[tex] @@ -34,7 +34,7 @@ \chapter{Example -- Global Trends in Obesity} \runExtCode{Rscript --save --restore}{Code/obesityPlot1.R}{obesityPlot} \begin{figure}[t] \begin{center} -\includegraphics{tmp/shareDelta.pdf} +\includegraphics{generated/shareDelta.pdf} \caption{The change in share-obese from 1975 to 2016 vs. the rate in 1975} \label{sharegain} \end{center} diff --git a/examples/QREMdoc/QREMdoc.pdf b/examples/QREMdoc/QREMdoc.pdf index f76810a..1763501 100644 Binary files a/examples/QREMdoc/QREMdoc.pdf and b/examples/QREMdoc/QREMdoc.pdf differ diff --git a/examples/QREMdoc/QREMdoc.tex b/examples/QREMdoc/QREMdoc.tex index ed3459d..02cac26 100644 --- a/examples/QREMdoc/QREMdoc.tex +++ b/examples/QREMdoc/QREMdoc.tex @@ -1,10 +1,11 @@ \documentclass[a4paper,10pt]{article} \usepackage[a4paper, total={6in, 10in}]{geometry} %\usepackage[square,sort,comma]{natbib} -\usepackage[R]{runcode} +\usepackage[R,stopserver]{../../runcode} +%\usepackage[cache]{../../runcode} \setminted[R]{fontsize=\footnotesize,linenos, frame=single, bgcolor=bg, breaklines=true} -% to compile it from scratch, create a tmp subfolder and comment out the [cache] options below. +% to compile it from scratch, create a generated subfolder and comment out the [cache] options below. % Keep in mind that some steps take a few minutes to complete (to run the bootstrap) @@ -30,23 +31,23 @@ \subsection{Assessing Goodness of Fit} We use this example to demonstrate two diagnostics functions in the package. \texttt{QRdiagnostics} is used to assess goodness of fit for one variable, at one selected quantile. For continuous variables it produces Q-Q plots, such that an appropriate model will results in the residual quantiles to lie along the main diagonal. For categorical variables it produces a spinogram, so that if the model is appropriate the percentage of points below the regression line is the chosen quantile for each level of the factor. The function \texttt{flatQQplot} can be used to plot a heatmap which represents the Q-Q plots for multiple quantiles in one graph. This can reveal whether a model fits well in all quantiles, or if it is misspecified in some. -\runExtCode{R}{Sim23.R}{tmp/sim23.tex}[cache] +\runR{Sim23.R}{sim23}[cache] In lines 21 and 22 we generate the diagnostic plot for the linear term, $x$, at $q=0.1$ for the correct and incorrect models, respectively. We see in Figure \ref{sim23} (a) that the points lie along the diagonal, indicating a good fit of the quadratic model, while in Figure \ref{sim23} (b) there is substantial deviation from the diagonal, suggesting that the linear model is inadequate. \begin{figure}[t!] \centering -\subfloat[\centering correct (quadratic) model]{\includegraphics[width=.4\linewidth]{tmp/sim23q10correct.pdf}} -\subfloat[\centering incorrect (linear) model]{\includegraphics[width=.4\linewidth]{tmp/sim23q10incorrect.pdf}} +\subfloat[\centering correct (quadratic) model]{\includegraphics[width=.4\linewidth]{generated/sim23q10correct.pdf}} +\subfloat[\centering incorrect (linear) model]{\includegraphics[width=.4\linewidth]{generated/sim23q10incorrect.pdf}} \caption{Simulation 23 -- Diagnostic plot using the \texttt{QRdiagnostics} function. The true model is $y \sim N(6x^2 + x +120, (0.1+0.5x)^2))$}\label{sim23} \end{figure} -In lines 23-24 we generate the `flat Q-Q plot' for the linear term, for all the quantiles in our analysis. Figure \ref{sim23flatqq} (a) shows a near-ideal situation. The heatmap represents the average difference between the x and y axes in the Q-Q plot. The range of x is partitioned and a goodness-of-fit test is applied to each segment, for each quantile. A green rectangle represents a range of x and a quantile for which the proportion (chi-squared) text is not significant, which means that for this range of x and for that quantile the model fits well (in the previous diagnostic plot it corresponds to having all points almost exactly along the main diagonal). The criticial values for the chi-squared test appear in the legend to the right of each flat QQ plot. Figure \ref{sim23flatqq} (b) shows that for all quantiles, the linear model is inadequate, since each column contains cells which represents differences between observed and expected counts which are quite different from 0. +In lines 23-24 we generate the `flat Q-Q plot' for the linear term, for all the quantiles in our analysis. Figure \ref{sim23flatqq} (a) shows a near-ideal situation. The heatmap represents the average difference between the x and y axes in the Q-Q plot. The range of x is partitioned and a goodness-of-fit test is applied to each segment, for each quantile. A green rectangle represents a range of x and a quantile for which the proportion (chi-squared) text is not significant, which means that for this range of x and for that quantile the model fits well (in the previous diagnostic plot it corresponds to having all points almost exactly along the main diagonal). The critical values for the chi-squared test appear in the legend to the right of each flat QQ plot. Figure \ref{sim23flatqq} (b) shows that for all quantiles, the linear model is inadequate, since each column contains cells which represents differences between observed and expected counts which are quite different from 0. \begin{figure}[t!] \centering -\subfloat[\centering correct (quadratic) model]{\includegraphics[width=.5\linewidth]{tmp/flatQQsim23q10correct.pdf}} -\subfloat[\centering correct (quadratic) model]{\includegraphics[width=.5\linewidth]{tmp/flatQQsim23q10incorrect.pdf}} +\subfloat[\centering correct (quadratic) model]{\includegraphics[width=.5\linewidth]{generated/flatQQsim23q10correct.pdf}} +\subfloat[\centering incorrect (linear) model]{\includegraphics[width=.5\linewidth]{generated/flatQQsim23q10incorrect.pdf}} \caption{Simulation 23 -- Diagnostic plot using the \texttt{flatQQplot} function.}\label{sim23flatqq} \end{figure} @@ -102,7 +103,7 @@ \subsection{Quantile Regression with Variable Selection} \begin{figure}[b!] \centering -\includegraphics[scale=0.5]{tmp/simVS5.pdf} +\includegraphics[scale=0.5]{generated/simVS5.pdf} \caption{Simulation 5 in Table A4 -- Diagnostic plot for the first predictor using the \texttt{QRdiagnostics} function.}\label{sim5vsqq} \end{figure} @@ -134,8 +135,8 @@ \section{Case Study -- Daily Temperature Data} \begin{figure}[t!] \centering -\subfloat[\centering 10th percentile of the minimum temperature]{\includegraphics[width=.5\linewidth]{tmp/LVmintemp.pdf}} -\subfloat[\centering 90th percentile of the maximum temperature]{\includegraphics[width=.5\linewidth]{tmp/LVmaxtemp.pdf}} +\subfloat[\centering 10th percentile of the minimum temperature]{\includegraphics[width=.5\linewidth]{generated/LVmintemp.pdf}} +\subfloat[\centering 90th percentile of the maximum temperature]{\includegraphics[width=.5\linewidth]{generated/LVmaxtemp.pdf}} \caption{Diagnostic plot for the Las Vegas temperature data.}\label{temps} \end{figure} diff --git a/examples/QREMdoc/Sim23.R b/examples/QREMdoc/Sim23.R index 046130a..5636b41 100644 --- a/examples/QREMdoc/Sim23.R +++ b/examples/QREMdoc/Sim23.R @@ -18,7 +18,7 @@ for (i in 1:length(qns)) { qrfits1[[i]] <- QREM(lm, linmod=y~x+I(x^2), df=dat, qn=qns[i]) qrfits2[[i]] <- QREM(lm, linmod=y~x, df=dat, qn=qns[i]) } -qrdg10c <- QRdiagnostics(dat$x, "x", qrfits1[[2]]$ui, qn=0.1, plot.it=TRUE, filename = "tmp/sim23q10correct.pdf") -qrdg10i <- QRdiagnostics(dat$x, "x", qrfits2[[2]]$ui, qn=0.1, plot.it=TRUE, filename = "tmp/sim23q10incorrect.pdf") -pvals <- flatQQplot(dat=dat, cnum = 2, qrfits=qrfits1, qns=qns, maxm = 20, plot.it = TRUE, filename = "tmp/flatQQsim23q10correct.pdf") -pvals <- flatQQplot(dat=dat, cnum = 2, qrfits=qrfits2, qns=qns, maxm = 20, plot.it = TRUE, filename = "tmp/flatQQsim23q10incorrect.pdf") \ No newline at end of file +qrdg10c <- QRdiagnostics(dat$x, "x", qrfits1[[2]]$ui, qn=0.1, plot.it=TRUE, filename = "generated/sim23q10correct.pdf") +qrdg10i <- QRdiagnostics(dat$x, "x", qrfits2[[2]]$ui, qn=0.1, plot.it=TRUE, filename = "generated/sim23q10incorrect.pdf") +pvals <- flatQQplot(dat=dat, cnum = 2, qrfits=qrfits1, qns=qns, maxm = 20, plot.it = TRUE, filename = "generated/flatQQsim23q10correct.pdf") +pvals <- flatQQplot(dat=dat, cnum = 2, qrfits=qrfits2, qns=qns, maxm = 20, plot.it = TRUE, filename = "generated/flatQQsim23q10incorrect.pdf") diff --git a/examples/QREMdoc/Sim24.R b/examples/QREMdoc/Sim24.R index 5cfa5c3..ccf765d 100644 --- a/examples/QREMdoc/Sim24.R +++ b/examples/QREMdoc/Sim24.R @@ -21,21 +21,21 @@ i <- 1 for (qn in qns) { qremFit <- QREM(lm, linmod=y~x*x2 +x3, df=data.frame(y,x,x2, x3), qn=qn) qrdg <- QRdiagnostics(x, "x",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F)) - dgx3 <- QRdiagnostics(x3, "x3",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F), filename="tmp/sim24q10correct.pdf") + dgx3 <- QRdiagnostics(x3, "x3",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F), filename="generated/sim24q10correct.pdf") for (j in 1:(L-1)) { qqp2[j,i] <- length(which(qrdg$y < xqs[j])) / length(which(qrdg$x < xqs[j])) } qremFit <- QREM(lm, linmod=y~x+x2+x3, df=data.frame(y,x,x2, x3), qn=qn) qrdg <- QRdiagnostics(x, "x",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F)) - dgx3 <- QRdiagnostics(x3, "x3",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F), filename="tmp/sim24q10incorrect.pdf") + dgx3 <- QRdiagnostics(x3, "x3",qremFit$ui, qn, plot.it = ifelse(qn == 0.1, T, F), filename="generated/sim24q10incorrect.pdf") for (j in 1:(L-1)) { qqp[j,i] <- length(which(qrdg$y < xqs[j])) / length(which(qrdg$x < xqs[j])) } i <- i+1 } -flatQQplot(x,xqs,qqp,qns) -flatQQplot(x,xqs,qqp2,qns) +#flatQQplot(x,xqs,qqp,qns) +#flatQQplot(x,xqs,qqp2,qns) library(xtable) qremFit <- QREM(lm, linmod=y~x*x2 +x3, df=data.frame(y,x,x2, x3), qn=qn) diff --git a/examples/QREMdoc/SimA4_5.R b/examples/QREMdoc/SimA4_5.R index 439374b..2ce145f 100644 --- a/examples/QREMdoc/SimA4_5.R +++ b/examples/QREMdoc/SimA4_5.R @@ -11,15 +11,15 @@ colnames(X) <- c("const", paste("X",1:5,sep="")) set.seed(11002) y <- X%*%as.matrix(coefs,ncol=1,nrow=6) + rnorm(n, 0, 0.1+X[,2]) dframe <- data.frame(y,X[,-1], matrix(rnorm((500-ncol(X)+1)*n,0,0.1), nrow=n, ncol=(500-ncol(X)+1))) -save(dframe, file="tmp/sim05.RData") +save(dframe, file="generated/sim05.RData") qn <- 0.25 -res <- QREM_vs("tmp/sim05.RData", 1, 2:501, qn=qn) +res <- QREM_vs("generated/sim05.RData", 1, 2:501, qn=qn) dfsemms <- dframe[,c(1, 1+res$fittedSEMMS$gam.out$nn)] qremFit <- QREM(lm, y~., dfsemms, qn=qn) ests <- rbind(qremFit$coef$beta, sqrt(diag(bcov(qremFit,linmod=y~., df=dfsemms, qn=0.2)))) rownames(ests) <- c("Estimate","s.d. (Bahadur)") library(xtable) -print(xtable(ests, digits=3), file="tmp/SimA4_5.tex") +print(xtable(ests, digits=3), file="generated/SimA4_5.tex") -qrdg <- QRdiagnostics(dfsemms[,2], "X1",qremFit$ui, qn, filename="tmp/simVS5.pdf") +qrdg <- QRdiagnostics(dfsemms[,2], "X1",qremFit$ui, qn, filename="generated/simVS5.pdf") diff --git a/examples/QREMdoc/temperatures.R b/examples/QREMdoc/temperatures.R index 05e46c6..0ed595e 100644 --- a/examples/QREMdoc/temperatures.R +++ b/examples/QREMdoc/temperatures.R @@ -13,10 +13,10 @@ linmodMin <- TMIN ~ sinedoy + year + (1|month) linmodMax <- TMAX ~ sinedoy + year + (1|month) qremFit10min <- QREM(lmer,linmodMin, dat,qn = 0.1) -qrdg10 <- QRdiagnostics(dat$year, "year (min. temp.)", qremFit10min$ui, qn=0.1, filename = "tmp/LVmintemp.pdf") +qrdg10 <- QRdiagnostics(dat$year, "year (min. temp.)", qremFit10min$ui, qn=0.1, filename = "generated/LVmintemp.pdf") qremFit90max <- QREM(lmer,linmodMax, dat,qn = 0.9) -qrdg90 <- QRdiagnostics(dat$year, "year (max. temp.)", qremFit90max$ui, qn=0.9, filename = "tmp/LVmaxtemp.pdf") +qrdg90 <- QRdiagnostics(dat$year, "year (max. temp.)", qremFit90max$ui, qn=0.9, filename = "generated/LVmaxtemp.pdf") # can also use a generalized additive model (gam), but the fit it slower # gam10fit <- QREM(gam, TMIN ~ s(yday,4) + year + month, data=dat, qn=0.1) diff --git a/examples/demoNode/classifiers.R b/examples/demoNode/classifiers.R index ca30c8a..2c94c76 100644 --- a/examples/demoNode/classifiers.R +++ b/examples/demoNode/classifiers.R @@ -5,12 +5,12 @@ library(e1071) library(kableExtra) loadData <- function() { - loc <- "http://archive.ics.uci.edu/ml/machine-learning-databases/" + loc <- "https://archive.ics.uci.edu/" ds <- "breast-cancer-wisconsin/breast-cancer-wisconsin.data" url <- paste(loc, ds, sep="") breast <- read.table(url, sep=",", header=FALSE, na.strings="?") names(breast) <- c("ID", "clumpThickness", "sizeUniformity", - "shapeUniformity", "maginalAdhesion", + "shapeUniformity", "marginalAdhesion", "singleEpithelialCellSize", "bareNuclei", "blandChromatin", "normalNucleoli", "mitosis", "class") df <- breast[-1] diff --git a/examples/demoNode/classifiers.jl b/examples/demoNode/classifiers.jl index 9d6c789..32cc73c 100644 --- a/examples/demoNode/classifiers.jl +++ b/examples/demoNode/classifiers.jl @@ -4,7 +4,7 @@ using Suppressor # load data -loc = "http://archive.ics.uci.edu/ml/machine-learning-databases/" +loc = "https://archive.ics.uci.edu/" ds = "breast-cancer-wisconsin/breast-cancer-wisconsin.data" url = loc * ds dat = readdlm(download(url), ',') @@ -12,7 +12,7 @@ keep = mapreduce(x->x, &, dat .!= "?", dims=2) df = convert(Array{Float64,2}, dat[vec(keep), 2:end]) df[:,10] = 1(df[:,10] .== 2.0) -# partion for traing and testing sets +# partition for training and testing sets Random.seed!(1) function partitionTrainTest(dat::Array{Float64,2}, at::Float64 = 0.7) n = size(dat)[1] diff --git a/examples/demoNode/index.htm b/examples/demoNode/index.htm index f9d9816..e499785 100644 --- a/examples/demoNode/index.htm +++ b/examples/demoNode/index.htm @@ -36,7 +36,7 @@

Demonstrating how to use the talk2stat package with NodeJS

for the back-end. The structure and functionality of the form is also entirely up to the user, and developing elaborate dashboards is possible.

The Wisconsin Breast Cancer dataset is available as a comma-delimited text file on the - UCI Machine Learning Server (http://archive.ics.uci.edu/ml). + UCI Machine Learning Server (https://archive.ics.uci.edu/). The dataset contains 699 fine-needle aspirate samples, where 458 (65.5%) are benign and 241 (34.5%) are malignant. The dataset contains a total of 11 variables and doesn’t include the variable names in the file. @@ -48,7 +48,7 @@

Demonstrating how to use the talk2stat package with NodeJS

In this example we will try different methods for predicting whether a patient has breast cancer from the characteristics of a fine-needle tissue aspiration (a tissue sample taken with a thin hollow needle from a lump or mass just under the skin).

-

Choose the type of classifier fromt he four available options, and the percentage of the data to be used for training.

+

Choose the type of classifier from the four available options, and the percentage of the data to be used for training.

diff --git a/examples/hw01_solution/face.pdf b/examples/hw01_solution/face.pdf new file mode 100644 index 0000000..43d25c7 Binary files /dev/null and b/examples/hw01_solution/face.pdf differ diff --git a/examples/hw01_solution/hw01_solution.pdf b/examples/hw01_solution/hw01_solution.pdf index b4af2f6..05bdb62 100644 Binary files a/examples/hw01_solution/hw01_solution.pdf and b/examples/hw01_solution/hw01_solution.pdf differ diff --git a/examples/hw01_solution/hw01_solution.tex b/examples/hw01_solution/hw01_solution.tex index 1d51596..bef85f8 100644 --- a/examples/hw01_solution/hw01_solution.tex +++ b/examples/hw01_solution/hw01_solution.tex @@ -18,9 +18,9 @@ \showCode{julia}{hw01-1.jl} \item \runJulia{hw01-2.jl}{prob2} - The correlation coefficient between $x$ and $y$ is $r=\inlnJulia{```print(corr)```}$, but the scatter plot looks like this: + The correlation coefficient between $x$ and $y$ is $r=\inlnJulia{```print(corr)```}[inln01]$, but the scatter plot looks like this: \begin{center} - \includegraphics[width=0.5\textwidth]{face.pdf} + \includegraphics[width=0.45\textwidth]{face.pdf} \end{center} The computation is performed using the following code \showCode{julia}{hw01-2.jl} diff --git a/generated/runcode_troubleshoot_inln2.tex b/generated/runcode_troubleshoot_inln2.tex new file mode 100644 index 0000000..52e3c12 --- /dev/null +++ b/generated/runcode_troubleshoot_inln2.tex @@ -0,0 +1 @@ +Error in Factorial(6) : could not find function "Factorial" diff --git a/generated/runcode_troubleshoot_inln4.tex b/generated/runcode_troubleshoot_inln4.tex new file mode 100644 index 0000000..6e1cb3c --- /dev/null +++ b/generated/runcode_troubleshoot_inln4.tex @@ -0,0 +1 @@ +**ZERO BYTES IN OUTPUT** diff --git a/generated/runcode_troubleshoot_inln5.tex b/generated/runcode_troubleshoot_inln5.tex new file mode 100644 index 0000000..6e1cb3c --- /dev/null +++ b/generated/runcode_troubleshoot_inln5.tex @@ -0,0 +1 @@ +**ZERO BYTES IN OUTPUT** diff --git a/generated/runcode_troubleshoot_inln6.tex b/generated/runcode_troubleshoot_inln6.tex new file mode 100644 index 0000000..a1708fb --- /dev/null +++ b/generated/runcode_troubleshoot_inln6.tex @@ -0,0 +1 @@ +720 diff --git a/generated/runcode_troubleshoot_inln7.tex b/generated/runcode_troubleshoot_inln7.tex new file mode 100644 index 0000000..67a8b14 --- /dev/null +++ b/generated/runcode_troubleshoot_inln7.tex @@ -0,0 +1,4 @@ + [,1] [,2] [,3] +[1,] 1 4 7 +[2,] 2 5 8 +[3,] 3 6 9 diff --git a/generated/runcode_troubleshoot_inln8.tex b/generated/runcode_troubleshoot_inln8.tex new file mode 100644 index 0000000..67a8b14 --- /dev/null +++ b/generated/runcode_troubleshoot_inln8.tex @@ -0,0 +1,4 @@ + [,1] [,2] [,3] +[1,] 1 4 7 +[2,] 2 5 8 +[3,] 3 6 9 diff --git a/generated/runcode_troubleshoot_inln9.tex b/generated/runcode_troubleshoot_inln9.tex new file mode 100644 index 0000000..7ed7bc7 --- /dev/null +++ b/generated/runcode_troubleshoot_inln9.tex @@ -0,0 +1 @@ +TIMED OUT+ diff --git a/generated/testWithrunR.tex b/generated/testWithrunR.tex new file mode 100644 index 0000000..f34ffbb --- /dev/null +++ b/generated/testWithrunR.tex @@ -0,0 +1,19 @@ + +Call: +lm(formula = y ~ x) + +Residuals: + Min 1Q Median 3Q Max +-1.5900 -0.8153 -0.1531 0.6379 2.8379 + +Coefficients: + Estimate Std. Error t value Pr(>|t|) +(Intercept) 0.95130 0.09629 9.88 <2e-16 *** +x 1.13879 0.10960 10.39 <2e-16 *** +--- +Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1 + +Residual standard error: 0.9626 on 98 degrees of freedom +Multiple R-squared: 0.5242, Adjusted R-squared: 0.5193 +F-statistic: 108 on 1 and 98 DF, p-value: < 2.2e-16 + diff --git a/paper/README.md b/paper/README.md new file mode 100644 index 0000000..a6acfbb --- /dev/null +++ b/paper/README.md @@ -0,0 +1,4 @@ +This folder contains the source and supplement files for the paper +[Reproducible Science with LaTeX](https://jds-online.org/journal/JDS/article/103/info), *J. Data Sci.* 2021, by Haim Bar and HaiYing Wang. + +The paper is based on this version of the `runcode`: [b1d05bf](https://github.com/Ossifragus/runcode/commit/b1d05bf5c897f00ec90c11c4da55a407b1612507). diff --git a/runcode.sty b/runcode.sty index ababf5a..63fdc73 100644 --- a/runcode.sty +++ b/runcode.sty @@ -4,132 +4,155 @@ % % This package is based on an ongoing work by Haim Bar and HaiYing Wang, and comments and questions are welcome! +\ProvidesPackage{runcode}[2025/01/17 runcode v2.4] + +\def \langs {} + \newif\ifruncode % Change to \runcodefalse if you want to suspend code execution \runcodetrue \DeclareOption{run}{\runcodetrue} \DeclareOption{cache}{\runcodefalse} +% to control whether \inln will get the result from the program(R, Python, etc.) +% or get it from cache (if available), then we define the following. The default +% is true (to run any \inln command in the tex file) +\newif\ifinlnrun +% Change to \inlnrunfalse if you want to get the results from cache for all +% \inln instances in the tex file (if cache is not available, it will be executed +% even if this variable is set to false) +\inlnruntrue + +% Choose the package for code typesetting. The default is minted. +% The other two options are listings and fvextra. For backward compatibility, +% it is possible to use nominted, in which case fvextra is used. \newif\ifminted \mintedtrue -\DeclareOption{nominted}{\mintedfalse} +\DeclareOption{nominted}{\mintedfalse} +\DeclareOption{minted}{\mintedtrue\fvextrafalse\listingsfalse} + +\newif\iflistings +\listingsfalse +\DeclareOption{listings}{\listingstrue\mintedfalse\fvextrafalse} + +\newif\iffvextra +\fvextrafalse +\DeclareOption{fvextra}{\fvextratrue\mintedfalse\listingsfalse} + + +\newif\ifreducedspace +\reducedspacefalse +\DeclareOption{reducedspace}{\reducedspacetrue} + % Some editors terminate all child processes after LaTeX compiling such as Emacs with Auctex. For this case, use the nohup option. It set the variable notnohup to be false, and the server will not be terminated by the parent process. \newif\ifnotnohup \notnohuptrue \DeclareOption{nohup}{\notnohupfalse} +\newcounter{portNo} +\setcounter{portNo}{65430} + +\NewDocumentCommand{\InitLang}{m}{ + % Create a configuration file for the server for the language provided + % in the argument, if it does not exist, and add to the serverslist.txt file. + \IfFileExists{#1.config}{}{ + \newwrite\tempfile + \immediate\openout\tempfile=#1.config + \immediate\write\tempfile{[SERVERCONFIG]} + \immediate\write\tempfile{PORT = \theportNo} + \immediate\write\tempfile{DEBUGFILE = #1debug.txt} + \immediate\write\tempfile{PIPETIMEOUT = 60} + \immediate\closeout\tempfile + \edef\langs{\langs#1_} + \stepcounter{portNo} + } + % Start the server. Need to run just once. Can comment out after the first + % compilation, but remember to terminate the server + \ifnotnohup + {\immediate\write18{python3 -c 'from talk2stat.talk2stat import server,client; server("./","#1") if not client("./","#1","``` ```") else print("server is already running")'}} + \else + {\immediate\write18{nohup python3 -c 'from talk2stat.talk2stat import server,client; server("./","#1") if not client("./","#1","``` ```") else print("server is already running")' &}} + \fi +} + \DeclareOption{R}{ - % create a configuration file for R server if it does not exist. - \IfFileExists{R.config}{}{ - \newwrite\tempfile - \immediate\openout\tempfile=R.config - \immediate\write\tempfile{[SERVERCONFIG]} - \immediate\write\tempfile{PORT = 65432} - \immediate\write\tempfile{DEBUGFILE = Rdebug.txt} - \immediate\write\tempfile{PIPETIMEOUT = 300} - \immediate\closeout\tempfile - } - % Start the server. Need to run just once. Can comment out after the first - % compilation, but remember to terminate the server - \ifnotnohup - {\immediate\write18{python3 -c 'from talk2stat.talk2stat import server,client; server("./","R") if not client("./","R","``` ```") else print("server is already running")'}} - \else - {\immediate\write18{nohup python3 -c 'from talk2stat.talk2stat import server,client; server("./","R") if not client("./","R","``` ```") else print("server is already running")' &}} - \fi + \InitLang{R} } \DeclareOption{julia}{ - % create a configuration file for R server if it does not exist. - \IfFileExists{julia.config}{}{ - \newwrite\tempfile - \immediate\openout\tempfile=julia.config - \immediate\write\tempfile{[SERVERCONFIG]} - \immediate\write\tempfile{PORT = 65431} - \immediate\write\tempfile{DEBUGFILE = juliadebug.txt} - \immediate\write\tempfile{PIPETIMEOUT = 300} - \immediate\closeout\tempfile - } - % Start the server. Need to run just once. Can comment out after the first - % compilation, but remember to terminate the server - \ifnotnohup - {\immediate\write18{python3 -c 'from talk2stat.talk2stat import server,client; server("./","julia") if not client("./","julia","``` ```") else print("server is already running")'}} - \else - {\immediate\write18{nohup python3 -c 'from talk2stat.talk2stat import server,client; server("./","julia") if not client("./","julia","``` ```") else print("server is already running")' &}} - \fi + \InitLang{julia} } \DeclareOption{matlab}{ - % create a configuration file for R server if it does not exist. - \IfFileExists{matlab.config}{}{ - \newwrite\tempfile - \immediate\openout\tempfile=matlab.config - \immediate\write\tempfile{[SERVERCONFIG]} - \immediate\write\tempfile{PORT = 65430} - \immediate\write\tempfile{DEBUGFILE = matlabdebug.txt} - \immediate\write\tempfile{PIPETIMEOUT = 300} - \immediate\closeout\tempfile - } - % Start the server. Need to run just once. Can comment out after the first - % compilation, but remember to terminate the server - \ifnotnohup - {\immediate\write18{python3 -c 'from talk2stat.talk2stat import server,client; server("./","matlab") if not client("./","matlab","``` ```") else print("server is already running")'}} - \else - {\immediate\write18{nohup python3 -c 'from talk2stat.talk2stat import server,client; server("./","matlab") if not client("./","matlab","``` ```") else print("server is already running")' &}} - \fi + \InitLang{matlab} } \DeclareOption{python}{ - % create a configuration file for Python server if it does not exist. - \IfFileExists{python.config}{}{ - \newwrite\tempfile - \immediate\openout\tempfile=python.config - \immediate\write\tempfile{[SERVERCONFIG]} - \immediate\write\tempfile{PORT = 65433} - \immediate\write\tempfile{DEBUGFILE = pythondebug.txt} - \immediate\write\tempfile{PIPETIMEOUT = 300} - \immediate\closeout\tempfile - } - % Start the server. Need to run just once. Can comment out after the first - % compilation, but remember to terminate the server - \ifnotnohup - {\immediate\write18{python3 -c 'from talk2stat.talk2stat import server,client; server("./","python") if not client("./","python","``` ```") else print("server is already running")'}} - \else - {\immediate\write18{nohup python3 -c 'from talk2stat.talk2stat import server,client; server("./","python") if not client("./","python","``` ```") else print("server is already running")' &}} - \fi + \InitLang{python} } + \DeclareOption{stopserver}{ \AtEndDocument{ - %% stop the server when the pdf compilation is done. - \immediate\write18{python3 -c 'from talk2stat.talk2stat import client; client("./","R","QUIT")'} - \immediate\write18{python3 -c 'from talk2stat.talk2stat import client; client("./","julia","QUIT")'} - \immediate\write18{python3 -c 'from talk2stat.talk2stat import client; client("./","matlab","QUIT")'} - \immediate\write18{python3 -c 'from talk2stat.talk2stat import client; client("./","python","QUIT")'} -}} + \IfFileExists{serverslist.txt}{ + \newread\file + \openin\file=serverslist.txt + \immediate\readline\file to\lang + \closein\file + \StrCount{\lang}{\detokenize{_}}[\langNo] + \newcounter{x} + \forloop{x}{0}{\value{x} < \langNo}{ + \StrBefore{\lang}{\detokenize{_}}[\curlang] + \StrBehind{\lang}{\detokenize{_}}[\lang] + %% stop the server when the pdf compilation is done. + \IfFileExists{\curlang.config}{ + \immediate\write18{python3 -c 'from talk2stat.talk2stat import client; client("./","\curlang","QUIT")'} + }{} + } + }{} + } +} \ProcessOptions* +\usepackage{etoolbox} \usepackage{morewrites} % allow more than 16 \write \usepackage[many]{tcolorbox} % to put boxes around text \tcbset{colframe=blue!25,colback=blue!10} % default colors for box output \usepackage{xcolor} % for highlighting -\usepackage[utf8x]{inputenc} +\usepackage{inputenc} \usepackage{textgreek} -\usepackage{filecontents} +% \usepackage{filecontents} \usepackage{xifthen} \usepackage{xparse} \usepackage{xstring} +\usepackage{forloop} + +%%%% check if \langs is empty !! +\ifdefempty{\langs}{}{ + \newwrite\outfile + \immediate\openout\outfile=serverslist.txt + \immediate\write\outfile{\langs} + \immediate\closeout\outfile} +{} -% Use minted if it is loaded; otherwise use fvextra +% Use minted by default; but the user can change to listings or fvextra. \ifminted \usepackage[cache=false]{minted} \setminted{fontsize=\footnotesize,breaklines=true} % minted default \else +\iflistings +\usepackage{listings} +\lstset{escapeinside=`',columns=fixed, extendedchars=true, basicstyle=\ttfamily\small, frame = lines} +\else \usepackage{fvextra} +\fvset{fontsize=\footnotesize, breaklines} +\fi \fi -\immediate\write18{mkdir tmp} +\newcommand{\generated}{generated} +\immediate\write18{mkdir -p \generated} \definecolor{bg}{rgb}{0.95,0.95,0.95} % code block background color @@ -161,9 +184,8 @@ \setvalue{\tmpname}{} % initialize with a blank - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% \showCode prints the source code, using minted for a pretty layout +% \showCode prints the source code, using minted or listings for a pretty layout % Arg #1 is the programming language, % Arg #2 is the source file name, % Args #3 and #4 are the first and last line to show (optional). @@ -173,22 +195,34 @@ \ifthenelse{\isempty{#3}}{ \ifminted \inputminted{#1}{#2} - \vspace{-\medskipamount} - \vspace{-\baselineskip} + \ifreducedspace + \vspace{-\medskipamount} + \vspace{-\baselineskip} + \fi \else - \VerbatimInput[fontsize=\footnotesize,linenos=true,frame=single,breaklines]{#2} + \iflistings + \lstinputlisting[language=#1]{#2} + \else + \VerbatimInput{#2} + \fi \fi }{ \ifminted \inputminted[firstline=#3, lastline=#4, firstnumber=1]{#1}{#2} - \vspace{-\medskipamount} - \vspace{-\baselineskip} + \ifreducedspace + \vspace{-\medskipamount} + \vspace{-\baselineskip} + \fi \else - \VerbatimInput[fontsize=\footnotesize,linenos=true,frame=single,breaklines, firstline=#3, lastline=#4, firstnumber=1]{#2} + \iflistings + \lstinputlisting[language=#1,firstline=#3, lastline=#4]{#2} + \else + \VerbatimInput[firstline=#3, lastline=#4, firstnumber=1]{#2} + \fi \fi } }{ - \textcolor{red}{\textbf{showCode: File #2 does not exist!}} + \textcolor{red}{\textbf{showCode: File $#2$ does not exist!}} } } @@ -202,13 +236,13 @@ % codeOutput is used). % Arg #4 controls when to run the code (optional - if not given, it listens % to \runcode; run = force the code to run; cache or anything else = -% use cache) +% use cache. If the output file does not exist, override the cache option and run the code) \NewDocumentCommand{\runExtCode}{m m m O{}}{ \IfFileExists{#2}{ \stepcounter{codeOutput} \ifthenelse{\isempty{#3}} - { \setvalue{\tmpname}{tmp/\jobname_tmp\thecodeOutput.tex} } - { \setvalue{\tmpname}{tmp/#3.tex} } + { \setvalue{\tmpname}{\generated/\jobname_tmp\thecodeOutput.tex} } + { \setvalue{\tmpname}{\generated/#3.tex} } % toggle \runcode above if you want to enable/disable code execution \ifthenelse{\isempty{#4}} { \ifruncode @@ -217,11 +251,13 @@ }{ \ifstrequal{#4}{run}{\immediate\write18{#1 #2 > \tmpname }}{} } + \IfFileExists{\tmpname}{}{\immediate\write18{#1 #2 > \tmpname }} }{ \textcolor{red}{\textbf{runExtCode: File #2 does not exist!}} } } + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % \includeOutput[Arg #1]{Arg #2} is used to embed the output from executed code. % Arg #1 is the output file name (optional - if not given, the counter @@ -231,8 +267,8 @@ \NewDocumentCommand{\includeOutput}{m O{vbox}}{\- \ifthenelse{\isempty{#1}} - {\setvalue{\tmpname}{tmp/\jobname_tmp\thecodeOutput.tex}} - {\unskip\setvalue{\tmpname}{tmp/#1.tex}\unskip}\unskip + {\setvalue{\tmpname}{\generated/\jobname_tmp\thecodeOutput.tex}} + {\unskip\setvalue{\tmpname}{\generated/#1.tex}\unskip}\unskip % even if the code is not executed, but we ran it before, we can use a % cached version if it exists. \IfFileExists{\tmpname} @@ -241,98 +277,242 @@ {\begin{tcolorbox} \ifminted\unskip \inputminted{text}{\tmpname}\unskip + \else\iflistings\unskip + \lstinputlisting{\tmpname}\unskip \else\unskip - \VerbatimInput[fontsize=\footnotesize,breaklines]{\tmpname}\unskip + \VerbatimInput{\tmpname}\unskip + \fi \fi \end{tcolorbox}} {\unskip\ifstrequal{#2}{inline} {\unskip\input{\tmpname}\unskip} {\input{\tmpname}}}} % if code execution disabled, and no cache: - {\begin{tcolorbox} - \textbf{Output file \tmpname~ not found}. Check the file name (it may be that the file name was given with the suffix .tex. If so, remove it). If the file name is correct the problem may be because code execution is disabled and no cache is available. If so, force the code to run again (using the [run] option). + {\begin{tcolorbox}[colback=red!5!white,colframe=red!75!black] + \textbf{Output file $\tmpname~$ not found}. Check the file name (it may be that the file name was given with the suffix .tex. If so, remove it). If the file name is correct the problem may be because code execution is disabled and no cache is available. If so, force the code to run again (using the [run] option). \end{tcolorbox}}} \NewDocumentCommand{\checkZeroBytes}{m}{ - \immediate\write18{python3 -c 'import os; print("ZERO BYTES IN OUTPUT", file=open("#1", "a")) if os.path.getsize("#1") == 0 else True'} + \immediate\write18{python3 -c 'import os; print("**ZERO BYTES IN OUTPUT**", file=open("#1", "a")) if os.path.getsize("#1") == 0 else True'} +} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% \writeChunk writes a selected chunk from source code as a .txt file. +% Arg #1 is the source file name, +% Arg #2 is the chunk identifier +% Args #3 and #4 are the beginning and end markings of a chunk (optional). + +\NewDocumentCommand{\writeChunk}{m m O{label===} O{===end}} { + \IfFileExists{#1}{ + \endlinechar=-1 + \def\printcode{0} % don't print until we see the begin-block string + \newread\file + \openin\file=#1 + \newwrite\outfile + \immediate\openout\outfile=\generated/#1-#2.txt + \loop\unless\ifeof\file + \immediate\readline\file to\fileline + \ifnum\printcode=1 + \IfSubStr{\fileline}{\detokenize{#4}}{ + \renewcommand{\printcode}{-1}}{\immediate\write\outfile{\fileline}} + \fi + \IfSubStr{\fileline}{\detokenize{#3}}{ + \StrBehind{\fileline}{\detokenize{#3}}[\chunkname] + \IfStrEq*{\detokenize{#2}}{\chunkname}{\renewcommand{\printcode}{1}}{} + } + {} + \repeat + \closein\file + \immediate\closeout\outfile + \endlinechar=13 + \ifnum\printcode=0 + \textcolor{red}{\textbf{Label $#2$ not found in #1}} + \fi + }{\textcolor{red}{\textbf{Source file $#1$ not found}}} +} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% \showChunk prints a selected chunk from source code, using minted or listings for a pretty layout. +% The chunk is identified in the source code by two strings that define the beginning and end of the chunk. +% The default beginning is label=== where should be a unique user-defined chunk ID. A label cannot include forward slash (/) +% The default end marker is ===end +% In the code, these markers should appear after a comment character, so that the code will run. +% Arg #1 is the programming language, +% Arg #2 is the source file name, +% Args #3 is the chunk identifier +% Args #4 and #5 are the beginning and end markings of a chunk (optional). + +\NewDocumentCommand{\showChunk}{m m m O{label===} O{===end}}{ + \IfFileExists{#2}{ + \immediate\write18{ad \generated/#2-#3.txt} + \writeChunk{#2}{#3}[#4][#5] + \IfFileExists{\generated/#2-#3.txt}{\showCode{#1}{\generated/#2-#3.txt}} + {\textcolor{red}{\textbf{Label $#3$ not found in #2}}} + }{\textcolor{red}{\textbf{Source file $#2$ not found}}} } %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% \inln{Arg #1}{Arg #2}[Arg #3] is used to execute short source code +% \inln{Arg #1}{Arg #2}[Arg #3][Arg #4] is used to execute short source code % and embed resulting output. % Arg #1 is the executable program; -% Arg #2 is the source code -% Arg #3 is the type output (if skipped or with empty value the default type is -% inline; vbox = verbatim in a box); - -\NewDocumentCommand{\inln}{m m O{inline}}{\- - \stepcounter{codeOutput}\unskip - \unskip\setvalue{\tmpname}{tmp/\jobname_inln\thecodeOutput}\unskip\unskip\unskip +% Arg #2 is the source code; +% Arg #3 is the output file name (optional). +% Arg #4 is the type output (if skipped or with empty value the default type is +% inline; vbox = verbatim in a box); If you append .cache to the argument +%. AND the output file in Arg#3 exists, then \inln will use the cached result; + +\NewDocumentCommand{\inln}{m m O{} O{inline}}{\- + \unskip + \ifthenelse{\isempty{#3}} + {\stepcounter{codeOutput}\unskip\unskip\setvalue{\tmpname}{\generated/\jobname_inln\thecodeOutput}\unskip\unskip\unskip} + {\unskip\setvalue{\tmpname}{\generated/#3}\unskip\unskip\unskip} + \unskip\unskip\unskip + \IfFileExists{\tmpname.tex}{\IfEndWith{#4}{cache}{\inlnrunfalse}{\inlnruntrue}}{\inlnruntrue} + \ifinlnrun \ifruncode % cache mode - don't try to run the code, just get the previous results \IfBeginWith{#2}{```}{\ifruncode\immediate\write18{#1 > \tmpname.tex}\unskip\fi} {\newwrite\tempfile \immediate\openout\tempfile=\tmpname.txt - \immediate\write\tempfile{#2} + \immediate\write\tempfile{\detokenize{#2}} \immediate\closeout\tempfile \ifruncode \immediate\write18{#1 \tmpname.txt > \tmpname.tex}\unskip \fi} - \fi % end cache mode + \fi + \fi% end cache mode + \unskip\unskip\unskip \IfFileExists{\tmpname.tex} {\checkZeroBytes{\tmpname.tex}\unskip - \ifstrequal{#3}{vbox} + \IfBeginWith{#4}{vbox} {\begin{tcolorbox} \ifminted\unskip \inputminted{text}{\tmpname.tex}\unskip + \else\unskip\iflistings\unskip + \lstinputlisting{\tmpname.tex}\unskip \else\unskip - \VerbatimInput[fontsize=\footnotesize,breaklines]{\tmpname.tex}\unskip + \VerbatimInput{\tmpname.tex}\unskip + \fi \fi \end{tcolorbox}} - {\unskip\input{\tmpname.tex}\unskip}} - {\textcolor{red}{\textbf{Output file \tmpname~ not found}}}} - -\NewDocumentCommand{\runR}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","R","#2")'} m m O{}} -{ - \def\runcmd{python3 -c 'from talk2stat.talk2stat import client; client("./","R","#2")'} - \ifstrequal{#1}{\runcmd} - {\runExtCode{#1}{}{#3}[#4]} - {\runExtCode{#1}{#2}{#3}[#4]} -} + {\unskip{\input{\tmpname.tex}}\unskip}} + {\textcolor{red}{\textbf{Output file $\tmpname~$ not found}}}} -\NewDocumentCommand{\inlnR}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","R","\tmpname.txt")'} m O{inline}}{\IfBeginWith{#2}{```}{\inln{python3 -c 'from talk2stat.talk2stat import client;client("./","R","#2")'}{#2}[#3]}{\inln{#1}{#2}[#3]}} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Extended and language-specific commands: -\NewDocumentCommand{\runJulia}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","julia","#2")'} m m O{}} -{ - \def\runcmd{python3 -c 'from talk2stat.talk2stat import client; client("./","julia","#2")'} - \ifstrequal{#1}{\runcmd} - {\runExtCode{#1}{}{#3}[#4]} - {\runExtCode{#1}{#2}{#3}[#4]} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% \runCodeIncOut: runs an external code and embeds the output. +% Arg #1 is the executable program, +% Arg #2 is the source file name, +% Arg #3 controls when to run the code (optional - if not given, it listens +% to \runcode; run = force the code to run; cache or anything else = +% use cache). The same functionality as that of Arg #4 of \runExtCode. +% Arg #4 is the output file name (optional - if not given, the counter +% codeOutput is used). The same functionality as that of Arg #4 of +% \runExtCode. +% Arg #5 is the optional type output with default "vbox" +% (vbox = verbatim in a box, tex = pure latex, or inline) +% The same functionality as that of Arg #2 of \includeOutput. + +\NewDocumentCommand{\runCodeIncOut}{m m O{} O{} O{vbox}}{ + \runExtCode{#1}{#2}{#4}[#3] + \includeOutput{#4}[#5] } -\NewDocumentCommand{\inlnJulia}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","julia","\tmpname.txt")'} m O{inline}}{\IfBeginWith{#2}{```}{\inln{python3 -c 'from talk2stat.talk2stat import client;client("./","julia","#2")'}{#2}[#3]}{\inln{#1}{#2}[#3]}} +\newcommand{\runcmd}[2] +{python3 -c 'from talk2stat.talk2stat import client; client("./","#1",#2)'} + +% a generic LANG - it will be used as a template for other languages +\newcommand{\LANG}{LANG} +\newcommand{\LANGcmd}{LANGcmd} -\NewDocumentCommand{\runMatlab}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","matlab","#2")'} m m O{}} -{ - \def\runcmd{python3 -c 'from talk2stat.talk2stat import client; client("./","matlab","#2")'} - \ifstrequal{#1}{\runcmd} - {\runExtCode{#1}{}{#3}[#4]} +\expandafter\NewDocumentCommand\csname run\LANG\endcsname +{O{} m m O{}} +{\ifthenelse{\isempty{#1}} + {\runExtCode{\runcmd{\LANGcmd}{"#2"}}{#2}{#3}[#4]} {\runExtCode{#1}{#2}{#3}[#4]} } -\NewDocumentCommand{\inlnMatlab}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","matlab","\tmpname.txt")'} m O{inline}}{\IfBeginWith{#2}{```}{\inln{python3 -c 'from talk2stat.talk2stat import client;client("./","matlab","#2")'}{#2}[#3]}{\inln{#1}{#2}[#3]}} +\expandafter\NewDocumentCommand\csname run\LANG IncOut\endcsname +{O{} m O{} O{} O{vbox}} +{\ifthenelse{\isempty{#1}} + {\runCodeIncOut{\runcmd{\LANGcmd}{"#2"}}{#2}[#3][#4][#5]} + {\runCodeIncOut{#1}{#2}[#3][#4][#5]} +} +\expandafter\NewDocumentCommand\csname inln\LANG\endcsname +{O{} m O{} O{inline}} +{\ifthenelse{\isempty{#1}} + {\IfBeginWith{#2}{```} + {\inln{\runcmd{\LANGcmd}{r"""\detokenize{#2}"""}}{#2}[#3][#4]} + {\inln{\runcmd{\LANGcmd}{"\tmpname.txt"}}{#2}[#3][#4]} + } + {\inln{#1}{#2}[#3][#4]} + \unskip\unskip\unskip +} -\NewDocumentCommand{\runPython}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","python","#2")'} m m O{}} -{ - \def\runcmd{python3 -c 'from talk2stat.talk2stat import client; client("./","python","#2")'} - \ifstrequal{#1}{\runcmd} - {\runExtCode{#1}{}{#3}[#4]} - {\runExtCode{#1}{#2}{#3}[#4]} +\expandafter\NewDocumentCommand\csname run\LANG Chunk\endcsname +{O{} m m O{} O{} O{vbox}} +{\IfFileExists{\generated/#2-#3.txt}{}{\writeChunk{#2}{#3}} + {\csname run\LANG IncOut\endcsname[#1]{\generated/#2-#3.txt}[#4][#2-#3][#6]} } -\NewDocumentCommand{\inlnPython}{O{python3 -c 'from talk2stat.talk2stat import client; client("./","python","\tmpname.txt")'} m O{inline}}{\IfBeginWith{#2}{```}{\inln{python3 -c 'from talk2stat.talk2stat import client;client("./","python","#2")'}{#2}[#3]}{\inln{#1}{#2}[#3]}} +% R +\NewDocumentCommand{\runR}{O{} m m O{}} +{{\renewcommand{\LANGcmd}{R}\runLANG[#1]{#2}{#3}[#4]}} + +\NewDocumentCommand{\runRIncOut}{O{} m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{R}\runLANGIncOut[#1]{#2}[#3][#4][#5]}} + +\NewDocumentCommand{\inlnR}{O{} m O{} O{inline}} +{{\renewcommand{\LANGcmd}{R}\inlnLANG[#1]{#2}[#3][#4]}} + +\NewDocumentCommand{\runRChunk}{O{} m m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{R}\runLANGChunk[#1]{#2}{#3}[#4][#5][#6]}} + +% Julia +\NewDocumentCommand{\runJulia}{O{} m m O{}} +{{\renewcommand{\LANGcmd}{julia}\runLANG[#1]{#2}{#3}[#4]}} + +\NewDocumentCommand{\runJuliaIncOut}{O{} m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{julia}\runLANGIncOut[#1]{#2}[#3][#4][#5]}} + +\NewDocumentCommand{\inlnJulia}{O{} m O{} O{inline}} +{{\renewcommand{\LANGcmd}{julia}\inlnLANG[#1]{#2}[#3][#4]}} + +\NewDocumentCommand{\runJuliaChunk}{O{} m m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{julia}\runLANGChunk[#1]{#2}{#3}[#4][#5][#6]}} + + +% Matlab +\NewDocumentCommand{\runMatLab}{O{} m m O{}} +{{\renewcommand{\LANGcmd}{matlab}\runLANG[#1]{#2}{#3}[#4]}} + +\NewDocumentCommand{\runMatLabIncOut}{O{} m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{matlab}\runLANGIncOut[#1]{#2}[#3][#4][#5]}} + +\NewDocumentCommand{\inlnMatLab}{O{} m O{} O{inline}} +{{\renewcommand{\LANGcmd}{matlab}\inlnLANG[#1]{#2}[#3][#4]}} + +\NewDocumentCommand{\runMatLabChunk}{O{} m m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{matlab}\runLANGChunk[#1]{#2}{#3}[#4][#5][#6]}} + + +% Python +\NewDocumentCommand{\runPython}{O{} m m O{}} +{{\renewcommand{\LANGcmd}{python}\runLANG[#1]{#2}{#3}[#4]}} + +\NewDocumentCommand{\runPythonIncOut}{O{} m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{python}\runLANGIncOut[#1]{#2}[#3][#4][#5]}} + +\NewDocumentCommand{\inlnPython}{O{} m O{} O{inline}} +{{\renewcommand{\LANGcmd}{python}\inlnLANG[#1]{#2}[#3][#4]}} + +\NewDocumentCommand{\runPythonChunk}{O{} m m O{} O{} O{vbox}} +{{\renewcommand{\LANGcmd}{python}\runLANGChunk[#1]{#2}{#3}[#4][#5][#6]}} %%%%%%%% @@ -343,17 +523,17 @@ \NewDocumentCommand{\runPythonBatch}{m m}{%\-\\ \stepcounter{codeOutput} \ifthenelse{\isempty{#2}} % set the output file name - { \setvalue{\tmpname}{tmp/\jobname_tmp\thecodeOutput.tex} } - { \setvalue{\tmpname}{tmp/#2.tex} } + { \setvalue{\tmpname}{\generated/\jobname_tmp\thecodeOutput.tex} } + { \setvalue{\tmpname}{\generated/#2.tex} } \IfFileExists{#1}{ \ifruncode \newwrite\tempfile - \immediate\openout\tempfile=tmp/\jobname_#1 + \immediate\openout\tempfile=\generated/\jobname_#1 \immediate\write\tempfile{import dill} \immediate\write\tempfile{from os.path import exists} - \immediate\write\tempfile{if exists('tmp/session'):} - \immediate\write\tempfile{ dill.load_session('tmp/session')} + \immediate\write\tempfile{if exists('\generated/session'):} + \immediate\write\tempfile{ dill.load_session('\generated/session')} \immediate\write\tempfile{} \newread\infile \openin\infile=#1 @@ -364,15 +544,15 @@ \repeat \endgroup \closein\infile - \immediate\write\tempfile{dill.dump_session('tmp/session')} + \immediate\write\tempfile{dill.dump_session('\generated/session')} \immediate\write\tempfile{} \immediate\closeout\tempfile - \immediate\write18{python3 tmp/\jobname_#1 > \tmpname} + \immediate\write18{python3 \generated/\jobname_#1 > \tmpname} \fi } { \immediate\write18{cat /dev/null > \tmpname} - \textcolor{red}{\textbf{runPythonBatch: File #1 does not exist!}} + \textcolor{red}{\textbf{runPythonBatch: File $#1$ does not exist!}} } } diff --git a/runcode_troubleshoot.pdf b/runcode_troubleshoot.pdf new file mode 100644 index 0000000..1c592c1 Binary files /dev/null and b/runcode_troubleshoot.pdf differ diff --git a/runcode_troubleshoot.tex b/runcode_troubleshoot.tex new file mode 100644 index 0000000..2d4d6c7 --- /dev/null +++ b/runcode_troubleshoot.tex @@ -0,0 +1,309 @@ +\documentclass[a4paper,10pt]{article} +\usepackage[a4paper, total={6in, 10in}]{geometry} +%\usepackage[square,sort,comma]{natbib} +% \usepackage[R,stopserver]{runcode} +\usepackage[cache]{runcode} + +\usepackage{caption} +%\usepackage{subcaption} +\usepackage{subfig} + + +\newcommand{\prob}[1] {\bigskip\noindent\textbf{\textcolor{orange}{Problem:}}\\#1\\\\} +\newcommand{\cause}[1] {\noindent\textit{\textcolor{blue}{Possible reason and solution:}}\\#1} +\newcommand{\cmnt}[1] {\bigskip\noindent\textit{Notes:} #1} + +% Title Page +\title{The runcode package -- troubleshooting} +\author{Haim Bar and HaiYing Wang} + + +\begin{document} +\maketitle + +When using the runcode package, you have to be aware of some usage rules, so this document attempts to anticipate all the possible user or system errors, and show how to interpret the output and fix the problems. + +\subsection*{Before you start using runcode} + +Before using the package make sure that you have Python (version 3.x) installed and in your path. Also, you must install the required \LaTeX~ packages: +morewrites, tcolorbox, xcolor, inputenc, textgreek, filecontents, xifthen, xparse, xstring, and fvextra. The minted package is optional but recommended. + +If you try to compile this file from its tex source, you will get errors (because the purpose of this document is to demonstrate errors and how to fix them.) It is assumed that you are using a cloned runcode repository, so files used in this document are ones in the directory structure as it is stored on github. + +The files generated by runcode live in the project's directory, so we assume the user has the necessary permissions to create, modify, delete files and subfolder. + +For the server mode the folder must contain a configuration file for each statistical language used (R, julia, matlab, Python). If such files don't exist, runcode will create them with default values. Remember to check if the defaults work for you. For example, you may need to change the port number, or increase the timeout parameter. + +\subsection*{Some possible errors and solutions} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\prob{runcode functions are not executed.} +\cause\bgroup +Check if you enabled the \verb|shell-escape| option when the document is compiled. If not, you will see warnings in the project's log file. For example: +\begin{Verbatim} +Package ifplatform Warning: + shell escape is disabled, so I can only detect \ifwindows. +! Package minted Error: You must invoke LaTeX with the -shell-escape flag. +\end{Verbatim} +\egroup + +\cause\bgroup +Check if the command-line tools you invoke from runcode are installed, and in your path (e.g., R, Julia, Matlab, Python). +\egroup + +\cmnt\bgroup +The runcode package can call any command-line function when it is used in `batch-mode'. That is, when the command-line tool is called separately each time a computation is performed from within the tex document (upon compilation). When using R, Julia, Python, or Matlab, the user can maintain a continuous session to the corresponding command-line tool. This saves initialization time, and allows to keep a session's history, for performing steps sequentially and efficiently. This is the recommended way to use runcode with R, Julia, Python, and Matlab. +At the present time, no other languages are supported for `server-mode' operation. +\egroup + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\prob{Code highlighting is not working properly.} +\cause\bgroup +When code is included in the manuscript, it is done via the \verb|\showCode| command. By default, the code-highlighting it done via the minted package. If code highlighting doesn't work, check if the minted package is installed properly. Python (3.x) also has to be installed, and also the Pygments package (which has to be installed via pip3). If you have trouble with the installation of minted, use the \verb|nominted| option when you include the runcode package. This will cause runcode to use the fvextra for code display, instead. +\egroup + + +\prob{Embedded code is not shown.} +\cause\bgroup +Check if you specified the source file correctly. +\verb|\showCode| prints the source code, using minted for a pretty layout. It takes 4 arguments. +Arg \#1 is the programming language, +Arg \#2 is the source file name, +Args \#3 and \#4 are the first and last line to show (optional). +If the source file name does not exist, you will get a red and bold error message. For example: +\begin{verbatim}\showCode{R}{Sim23.R} +\end{verbatim} +\showCode{R}{Sim23.R} +\noindent +In contrast, when the file exists, as in this example +\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R} +\end{verbatim} +the file will be shown correctly: +\showCode{R}{paper/supplement/Code/code1.R} +\egroup + +\cause\bgroup +If the programming language is misspecified or not recognized by minted or fvextra, the code highlighting may not be shown correctly. +\begin{verbatim}\showCode{matlab}{paper/supplement/Code/code1.R} +\end{verbatim} +\showCode{matlab}{paper/supplement/Code/code1.R} +\egroup + +\cause\bgroup +If the line number in Arg \#3 exceeds the actual number of lines in the code, the code box will be empty, and the \LaTeX~ compiler will show an error message in its log file (`Empty verbatim environment'). For example: +\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R}[6][8] +\end{verbatim} +% \showCode{R}{paper/supplement/Code/code1.R}[6][8] +% \vspace{-2em} +\noindent Some \LaTeX~ compilers will stop the compilation when they encounter `Empty verbatim environment' but will allow you to manually continue the compilation (and the generated pdf will contain an empty box with no code in it.) + +\noindent If the number in Arg. \#4 is greater than the number of lines in the file, minted will show the code up to the last line (so this misspecification is harmless). +\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R}[4][8] +\end{verbatim} +\showCode{R}{paper/supplement/Code/code1.R}[4][8] +\egroup + +\cmnt{Remember that the compiler is case-sensitive, so test.R is not the same as Test.R.} + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\prob{Errors when running code.} +\cause\bgroup +In batch-mode with \verb|\runExtCode|, if the source file name does not exist, you will get a red and bold error message:\\ +\verb|\runExtCode{julia}{test2.jl}{test2}|\\ +\runExtCode{julia}{test2.jl}{test2}\\ +Again, check for spelling errors in the file name, which is the most common reason for such problems.\\ + +\noindent Similarly, when using the \textit{server-mode} of runcode, we can use the shortcuts to R, Julia, Python, or Matlab instead of \verb|\runExtCode|. The usage is similar, but the language name is inferred from the command. For example, we can have:\\ +\verb|\runR{paper/supplement/Code/code1.R}{testWithrunR}| +\runR{paper/supplement/Code/code1.R}{testWithrunR}\\ +If the source code file doesn't exist, we get an error message as with \verb|\runExtCode|\\ +\verb|\runJulia{test2.jl}{test2}|\\ +\runJulia{test2.jl}{test2}\\ +\egroup + +\cause\bgroup +Check that the correct executable is used, since \verb|\runExtCode| requires the specific program and command line arguments. For example, using R as the executable will not work in the following example:\\ +\verb|\runExtCode{R}{paper/supplement/Code/code1.R}{testWithR}|\\ +%\runExtCode{R}{paper/supplement/Code/code1.R}{testWithR} +It will create the file generated/testWithR.tex, and in it you will see\\ +\verb|ARGUMENT 'paper/supplement/Code/code1.R' __ignored__|\\ +This is a fatal error -- the compiler will get stuck because it will wait for the code to finish, and the compilation process will have to be terminated by the user. +\noindent +The correct way to use \verb|\runExtCode| with R in batch-mode is\\ +\verb|\runExtCode{Rscript --save --restore}{paper/supplement/Code/code1.R}{test}|\\ +See the package's documentation for more working examples.\\ +\egroup + + +\cause\bgroup +Check for syntax errors int the code. +In the \textit{server-mode} only, \verb|\inln|, \verb|\inlnR|, \verb|\inlnJulia|, \verb|\inlnPython|, and \verb|\inlnMatlab| commands can be used to execute short source code and embed the resulting output within the text. It takes 3 arguments. +Arg \#1 is the executable program; +Arg \#2 is the source code +Arg \#3 is the type output (if skipped or with empty value the default type is inline; vbox = verbatim in a box). + +\noindent If the code contains a syntax error, the error produced by the statistical software will be embedded in the text. For example, if we use Factorial, instead of the real function (factorial) we get an error message:\\ +\verb|The factorial of 6 is \inlnR{```cat(Factorial(6))```}.|\\ +would produce the following:\\ +The factorial of 6 is \inlnR{```cat(Factorial(6))```}.\\ + +\noindent Such errors are not due to \LaTeX~ syntax or compilation, so runcode doesn't highlight them. Automatically identifying and highlighting such errors would require case-by-case analysis of the output of specific command-line tools. For now, it's up to the user to check the code before it is embedded in the tex document, and to check the output after the compilation. +\egroup + + +\cmnt\bgroup +Recall that \verb|\runExtCode| which is used to run an external code takes 4 arguments: +Arg \#1 is the executable program, +Arg \#2 is the source file name, +Arg \#3 is the output file name (optional - if not given, the counter + codeOutput is used). +Arg \#4 controls when to run the code (optional - if not given, it listens + to \verb|\runcode|; run = force the code to run; cache or anything else = + use cache).\\ +The first argument can be \textit{any} command-line executable, including user-defined program names (compiled code, aliases, etc.). Because of that, we do not perform validity check before trying to execute it. +For example, the following will not produce any error message:\\ +\verb|\runExtCode{C}{test.R}{}|\\ +but it will be possible to see an empty file in the generated folder. This will be obvious to the user once the \verb|\includeOutput| command is executed in order to embed results in the compiled pdf document. +\egroup + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\prob{Output is not produced, or unexpected/incorrect results are shown.} +\cause\bgroup +This could happen due to incorrect usage of the \verb|\includeOutput| function, which is used to embed the output from executed code. It takes 2 arguments: +Arg \#1 is the output file name (optional - if not given, the counter codeOutput is used). +Arg \#2 is the optional type output with default "vbox" (vbox = verbatim in a box, tex = pure latex, or inline). +For example, if we run the code\\ +\verb|\runR{paper/supplement/Code/code1.R}{testWithrunR}| +\runR{paper/supplement/Code/code1.R}{testWithrunR}\\ +\noindent +Then to see the output, the correct usage is\\ +\verb|\includeOutput{testWithrunR}[vbox]|\\ +\includeOutput{testWithrunR}[vbox] + +\noindent Although the physical file is named generated/testWithrunR.tex, we drop the generated/ prefix and the .tex suffix. If we do include them, the file will not be found and we'll get an error message:\\ +\verb|\includeOutput{testWithrunR.tex}[vbox]| +\includeOutput{testWithrunR.tex}[vbox] +\egroup + +\cmnt\bgroup +If anything other than vbox, tex, or inline is provided in the square brackets when using \verb|\includeOutput|, the output will be embedded in the document as plain text. While no error or warning is raised, this may cause problems in the compilation of the tex file and therefore should be avoided. (For example, if the output contains underscores then the compiler will report an error because it would appear that math symbols are used in text mode).\\ +\egroup + + +\cause\bgroup +When using one of the \verb|inln| functions, the code does not produce any output. +In some cases it is perfectly fine if embedded code does not produce output, but when using \verb|\inln| this is not the case, so runcode checks if the command used within \verb|\inln| produced a zero-byte output file. If it did, runcode will show an appropriate message, like in the following example. Note that the reason no output is produced is that file.csv does not exist.\\ +\verb|The number of columns is \inlnR{```dat <- read.csv("file.csv"); cat(ncol(dat))```}.|\\ +This will result in:\\ +The number of columns is \inlnR{```dat <- read.csv("file.csv"); cat(ncol(dat))```}.\\ +\egroup + +\cause\bgroup +When using one of the \verb|inln| functions, more than one line of output is produced by the code. +The \verb|\inln| commands are designed to put a single string in a line, so the code which produces the output should not include a new line. For example, the following is incorrect:\\ +\verb|The factorial of 6 is \inlnR{```cat(factorial(6),"\\n")```}.|\\ +The factorial of 6 is \inlnR{```cat(factorial(6),"\\n")```}. +\noindent +The correct way is:\\ +\verb|The factorial of 6 is \inlnR{```cat(factorial(6))```}.|\\ +which produces the following: +The factorial of 6 is \inlnR{```cat(factorial(6))```}. + + +\noindent +Similarly, if we want to print more than a one-line string, we should use the vbox option (or use \verb|\includeOutput|.) For example,\\ +\verb|A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}.|\\ +Will appear like this: A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}. +\\ +\verb|A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}[vbox].|\\ +Will look better: A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}[vbox]. +\egroup + + +\cause\bgroup +This type of problem can also be due to invalid output in the cache files. If for some reason, the cache files contain special characters or an underscore in text-mode, then the output will be corrupted. It is recommended to clear the cache, and if server-mode is used to also restart the server. For example, suppose that the directory of the project is called proj, and we're using Python in server-mode. From the command line, run the following: +\begin{Verbatim} +cd proj +rm -f proj.aux generated/*.txt nohup.out proj.synctex.gz +python3 -c 'from talk2stat.talk2stat import client; client("./","python","QUIT")' +rm -f serverPIDpython.txt pythondebug.txt talk2stat.log +\end{Verbatim} +Then, recompile the project.\\ +\egroup + + +\cause\bgroup +Invalid, missing, or unexpected output can be due to having another runcode running on the system, but in another directory. If this happens, you will see a message in the log file saying that the server is already running. When multiple projects attempt to connect with the statistical software through the same port, only the first invocation will succeed but the subsequent ones will be connected to the first one. This will create unexpected results for all projects using the same port. Suppose your new project is B, and the runcode server of project A is still running and using the same port. In this case you may, for example, +\begin{itemize} +\item Accidentally overwrite variables in project A, or use the wrong ones in project B if they were also defined in A; +\item Try to run code which is stored in project B, but you get an error message in the pdf file such as `\texttt{cannot open file 'mycode.R': No such file or directory}'. The reason for this error is that the other instance of the runcode server uses project A's base directory, and the file mycode.R is not in the right path. +\end{itemize} +To fix it, you can do one of the following: +\begin{enumerate} +\item configure each project to use another port to communicate with talk2stat (which will prevent any conflict between the projects), or, +\item you can stop all instances of talk2stat running on the system before compiling your current project. +\end{enumerate} +To do the latter, find all the talk2stat processes, and find the corresponding directories. For example, suppose that multiple projects use talk2stat to communicate with Julia. From the command-line, do the following: +\begin{Verbatim} +cd proj +ps -ef | grep talk2stat # get all the PIDs. E.g., 36797 +# on Mac: +lsof -a -d cwd -p 36797 +# on Linux: +pwdx 36797 +# lsof or pwdx will give you the directory name of the other projects which run talk2stat +# cd to each directory, and run: +python3 -c 'from talk2stat.talk2stat import client; client("./","julia","QUIT")' +rm -f serverPIDjulia.txt juliadebug.txt talk2stat.log +\end{Verbatim} +\egroup + + +\cause\bgroup +If the correct output is in the files in generated directory, but it's not showing in the pdf file, it can be due to the \LaTeX~ compiler's naming of temporary files. Usually, the auto-generated file names start with the main tex file name (e.g. if the main tex file is called troubleshoot.tex then runcode will create files such as \verb|troubleshoot_inln5.tex|). However, working with Overleaf we noticed that their compiler uses another convention for temporary file names. To prevent such problems, you can add the following in the preamble of the main document: +\begin{Verbatim} +\edef\TeXjobname{\jobname} % (this line is not really essential.) +\edef\jobname{\detokenize{troubleshoot}} +\end{Verbatim} +\egroup + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +\prob{Working in server-mode, the server is stopped after each compilation.} +\cause\bgroup +If you are using the server-mode, be aware that some editors terminate all child processes at the end of \LaTeX compilation. For example, Emacs with Auctex behaves this way. For this case, use the nohup option, and the server will not be terminated by the parent process.\\ +If you want the server to be stopped after each compilation (regardless of the editor you are using), use the stopserver option. While you are compiling the tex document often, you may want to keep the server side running in order to save time during initialization, and to maintain the variables and results already in memory. However, it is a good idea to use the stopserver option when you are done, just to prevent any conflicts (see above).\\ +It's recommended to check that port numbers are unique, and when a project is not expected to be compiled for a while to enable the stopserver option. +\egroup + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +\prob{The process seems to hang during code execution when the \LaTeX~ document is compiled.} +\cause\bgroup +We mentioned that code being sent to the software must be syntactically valid. In addition, keep in mind that the server waits for complete commands. So, for example, if we try\\ +\verb|The factorial of 6 is \inlnR{```cat(factorial(6)```}.|\\ +it will cause the server to hang because it awaits the closing parenthesis. Eventually, it will time out but the code cannot be executed, so the desired output will not be produced, and an error message will replace the output, as in the following example. The timeout interval is set in the config files for R, Julia, Python, and Matlab. +\verb|The factorial of 6 is \inlnR{```cat(factorial(6)```}.|\\ +The factorial of 6 is \inlnR{```cat(factorial(6)```}.\\ +In this example the cat statement is missing a closing parenthesis.\\ +\egroup + + + +\end{document} + + diff --git a/troubleshoot.pdf b/troubleshoot.pdf deleted file mode 100644 index 55fefe6..0000000 Binary files a/troubleshoot.pdf and /dev/null differ diff --git a/troubleshoot.tex b/troubleshoot.tex deleted file mode 100644 index 7020e93..0000000 --- a/troubleshoot.tex +++ /dev/null @@ -1,198 +0,0 @@ -\documentclass[a4paper,10pt]{article} -\usepackage[a4paper, total={6in, 10in}]{geometry} -%\usepackage[square,sort,comma]{natbib} -\usepackage[R,stopserver]{runcode} - -\usepackage{caption} -%\usepackage{subcaption} -\usepackage{subfig} - - -% Title Page -\title{The runcode package -- troubleshooting} -\author{Haim Bar and HaiYing Wang} - - -\begin{document} -\maketitle - -When using the runcode package, you have to be aware of some usage rules, so this document attempts to anticipate all the possible user or system errors, and show how to interpret the output and fix the problems. - -\subsection*{Before you start using runcode} - -Before using the package make sure that you have Python (version 3.x) installed and in your path. Also, you must install the required \LaTeX~ packages: -morewrites, tcolorbox, xcolor, inputenc, textgreek, filecontents, xifthen, xparse, xstring, and fvextra. The minted package is optional but recommended. - -If you try to compile this file from its tex source, you will get errors (because the purpose of this document is to demonstrate errors and how to fix them.) It is assumed that you are using a cloned runcode repository, so files used in this document are ones in the directory structure as it is stored on github. - -The files generated by runcode live in the project's directory, so we assume the user has the necessary permissions to create, modify, delete files and subfolder. - -For the server mode the folder must contain a configuration file for each statistical language used (R, julia, matlab, Python). If such files don't exist, runcode will create them with default values. Remember to check if the defaults work for you. For example, you may need to change the port number, or increase the timeout parameter. - - -\subsection*{The preamble} -Several options are available via the \verb|\usepackage[options]{runcode}| statement in the preamble. See the package's documentation for details. Here, we focus on troubleshooting only. - -First and foremost, if it appears that the runcode functions are not executed, be sure to turn on the \verb|shell-escape| option. The main log file will contain a relevant warning if this option is not enabled. - -Equally important, the command-line tools you intend to call from runcode must be installed, and in your path (e.g., R, Julia, Matlab, Python). - -The runcode package can call any command-line function when it is used in `batch-mode'. That is, when the command-line tool is called separately each time a computation is performed from within the tex document (upon compilation). When using R, Julia, or Matlab, the user can maintain a continuous session to the corresponding command-line tool. This saves initialization time, and allows to keep a session's history, for performing steps sequentially. This is the recommended way to use runcode with R, Julia, and Matlab. -At the present time, no other languages are supported for `server-mode' operation. - -When code is included in the manuscript, it is done via the \verb|\showCode| command (see below). By default, the code-highlighting it done via the minted package. If code highlighting doesn't work, check the the minted package is installed properly. Python (3.x) also has to be installed. If you have trouble with the installation of minted, use the \verb|nominted| option. This will cause runcode to use the fvextra for code display, instead. - -If you are using the server-mode, be aware that some editors terminate all child processes at the end of \LaTeX compilation. For example, Emacs with Auctex behaves this way. For this case, use the nohup option, and the server will not be terminated by the parent process. - -If you want the server to be stopped after each compilation, use the stopserver option. While you are compiling the tex document often, you may want to keep the server side running in order to save time during initialization, and to maintain the variables and results already in memory. However, it is a good idea to use the stopserver option when you are done, just to prevent any confusion when using runcode for multiple projects. Such confusion may only occur if the configuration file for two files in two different projects (say, A and B) use the same port number. Suppose your new project is B, and the runcode server of project A is still running and using the same port. In this case you may, for example, -\begin{itemize} -\item Accidentally overwrite variables in project A, or use the wrong ones in project B if they were also defined in A; -\item Try to run code which is stored in project B, but you get an error message in the pdf file such as `\texttt{cannot open file 'mycode.R': No such file or directory}'. The reason for this error is that the other instance of the runcode server uses project A's base directory, and the file mycode.R is not in the right path. -\end{itemize} -It's best to check that port numbers are unique, and when a project is not expected to be compiled for a while to enable the stopserver option. - - -\subsection*{The showCode command} -\verb|\showCode| prints the source code, using minted for a pretty layout. It takes 4 arguments. -Arg \#1 is the programming language, -Arg \#2 is the source file name, -Args \#3 and \#4 are the first and last line to show (optional). - -If the source file name does not exist, you will get a red and bold error message (remember that the compiler is case-sensitive, so test.R is not the same as Test.R). -\begin{verbatim}\showCode{R}{Sim23.R} -\end{verbatim} -\showCode{R}{Sim23.R} - -\bigskip -In contrast, when the file exists, as in this example -\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R} -\end{verbatim} -the file will be shown correctly: -\showCode{R}{paper/supplement/Code/code1.R} - - -If the programming language is misspecified or not recognized by minted or fvextra, the code highlighting may not be shown correcly. -\begin{verbatim}\showCode{matlab}{aper/supplement/Code/code1.R} -\end{verbatim} -\showCode{matlab}{paper/supplement/Code/code1.R} - -If the line number in Arg \#3 exceeds the actual number of lines in the code, the code box will be empty, and the \LaTeX~ compiler will show an error message in its log file (`Empty verbatim environment'). -\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R}[6][8] -\end{verbatim} -\showCode{R}{paper/supplement/Code/code1.R}[6][8] - -If the number in Arg. \#4 is greater than the number of lines in the file, minted will show the code up to the last line (so this misspecification is harmless). -\begin{verbatim}\showCode{R}{paper/supplement/Code/code1.R}[4][8] -\end{verbatim} -\showCode{R}{paper/supplement/Code/code1.R}[4][8] - -\subsection*{The runExtCode command} - - -\verb|\runExtCode| runs an external code. It takes 4 arguments. -Arg \#1 is the executable program, -Arg \#2 is the source file name, -Arg \#3 is the output file name (optional - if not given, the counter - codeOutput is used). -Arg \#4 controls when to run the code (optional - if not given, it listens - to \verb|\runcode|; run = force the code to run; cache or anything else = - use cache). - -If the source file name does not exist, you will get a red and bold error message:\\ -\verb|\runExtCode{julia}{test2.jl}{test2}|\\ -\runExtCode{julia}{test2.jl}{test2} - -The first argument can be any command-line executable, including user-defined program names (compiled code, aliases, etc.). Because of that, we do not perform validity check before trying to execute it. -For example, the following will not produce any error message:\\ -\verb|\runExtCode{C}{test.R}{}|\\ -but it will be possible to see an empty file in the tmp folder. This will be obvious to the user once the \verb|\includeOutput| command is executed (see below). - -Note that \verb|\runExtCode| requires the specific program and command line arguments. For example, using R as the language will not work:\\ -\verb|\runExtCode{R}{paper/supplement/Code/code1.R}{testWithR}|\\ -\runExtCode{R}{paper/supplement/Code/code1.R}{testWithR} -It will create the file tmp/testWithR.tex, and in it you will see\\ -\verb|ARGUMENT 'paper/supplement/Code/code1.R' __ignored__| - -The correct way to use \verb|\runExtCode| with R is\\ -\verb|\runExtCode{Rscript --save --restore}{paper/supplement/Code/code1.R}{test}|\\ -See the package's documentation for more working examples. - -\subsection*{The runR, runJulia, runMatlab commands} -When using the \textit{server-mode} of runcode, we can use the shortcuts to R, Julia, or Matlab instead of \verb|\runExtCode|. The usage is similar, but the language name is inferred from the command. For example, we can have:\\ -\verb|\runR{paper/supplement/Code/code1.R}{testWithrunR}| -\runR{paper/supplement/Code/code1.R}{testWithrunR}\\ - -If the source code file doesn't exist, we get an error message as with \verb|\runExtCode|\\ -\verb|\runJulia{test2.jl}{test2}|\\ -\runJulia{test2.jl}{test2} - - - -\subsection*{The includeOutput command} -\verb|\includeOutput| is used to embed the output from executed code. It takes 2 arguments: -Arg \#1 is the output file name (optional - if not given, the counter codeOutput is used). -Arg \#2 is the optional type output with default "vbox" (vbox = verbatim in a box, tex = pure latex, or inline). - -We first have to run the code\\ -\verb|\runR{paper/supplement/Code/code1.R}{testWithrunR}| -\runR{paper/supplement/Code/code1.R}{testWithrunR} - -To see the output, we use\\ -\verb|\includeOutput{testWithrunR}[vbox]|\\ -\includeOutput{testWithrunR}[vbox] - -Although the physical file is named tmp/testWithrunR.tex, we drop the tmp/ prefix and the .tex suffix. If we do include them, the file will not be found and we'll get an error message:\\ -\verb|\includeOutput{testWithrunR.tex}[vbox]| -\includeOutput{testWithrunR.tex}[vbox]\\ - -If anything other than vbox, tex, or inline is provided in the brackets, the output will be embedded in the document in plain text. While no error or warning is raised, this may cause problems in the compilation of the tex file and therefore should be avoided. (For example, if the output contains underscores then the compiler will report an error because it would appear that math symbols are used in text mode). - - -\subsection*{The inln, inlnR, inlnJulia, and inlnMatlab commands} -In the \textit{server-mode} only, \verb|\inln| is used to execute short source code and embed resulting output within the text. It takes 3 arguments. -Arg \#1 is the executable program; -Arg \#2 is the source code -Arg \#3 is the type output (if skipped or with empty value the default type is inline; vbox = verbatim in a box). - -The syntax of the code has to be correct. If it's not, the error from the statistical software will be embedded in the text. For example, if we use Factorial, instead of the real function (factorial) we get an error message.\\ -\verb|The factorial of 6 is \inlnR{```cat(Factorial(6))```}.|\\ -would produce the following:\\ -The factorial of 6 is \inlnR{```cat(Factorial(6))```}. - -Such errors are not due to \LaTeX~ syntax or compilation, so runcode doesn't highlight them. Automatically identifying and highlighting such errors would require case-by-case analysis of the output of specific command-line tools. For now, it's up to the user to check the code before it is embedded in the tex document, and to check the output after the compilation. - -In some cases it is perfectly fine if embedded code does not produce output, but when using \verb|\inln| this is not the case, so we check if the command used within \verb|\inln| produced a zero-byte output file. If it did, we show an appropriate message, like in the following example. Note that the reason no output is produced is that file.csv does not exist.\\ -\verb|The number of columns is \inlnR{```dat <- read.csv("file.csv"); cat(ncol(dat))```}.|\\ -This will result in:\\ -The number of columns is \inlnR{```dat <- read.csv("file.csv"); cat(ncol(dat))```}. - -The \verb|\inln| command is designed to put a single string in a line, so the code which produces the output should not include a new line. For example, the correct way is:\\ -\verb|The factorial of 6 is \inlnR{```cat(factorial(6))```}.| -The factorial of 6 is \inlnR{```cat(factorial(6))```}.\\ -The following is incorrect:\\ -\verb|The factorial of 6 is \inlnR{```cat(factorial(6),"\\n")```}.|\\ -The factorial of 6 is \inlnR{```cat(factorial(6),"\\n")```}. - -Similarly, if we want to print more than a one-line string, we should use the vbox option (or use \verb|\includeOutput|.) For example, - -\verb|A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}.|\\ -Will appear like this: A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}. - -\verb|A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}[vbox].|\\ -Will look better: A matrix \inlnR{```matrix(1:9,ncol=3,nrow=3)```}[vbox]. - -\subsection*{Other potential errors} -We mentioned that code being sent to the software must be syntactically valid. In addition, keep in mind that the server waits for complete commands. So, for example, if we try\\ -\verb|The factorial of 6 is \inlnR{```cat(factorial(6)```}.|\\ -it will cause the server to hang because it awaits the closing parenthesis. Eventually, it will time out but the code cannot be executed, so the desired output will not be produced. The timeout interval is set in the config files for R, Julia, and Matlab. -For example, in the following the cat statement is missing a closing parenthesis.\\ -\verb|The factorial of 6 is \inlnR{```cat(factorial(6)```}.|\\ -The factorial of 6 is \inlnR{```cat(factorial(6)```}. - - - -% texcount -char -1 -sum tmp/testruncode_inln4.tex - -\end{document} - -