trinROC update

trinROC/DESCRIPTION

 Package: trinROC
 Title: Statistical Tests for Assessing Trinormal ROC Data
-Version: 0.5
-Date: 2021-01-04
+Version: 0.5.1
+Date: 2021-06-29
 Authors@R: c(person("Samuel", "Noll", email = "uncle.sam@gmx.net", role = c("aut")), person("Reinhard", "Furrer", role = c("cre", "aut"), email = "reinhard.furrer@math.uzh.ch"), person("Benjamin", "Reiser", role = c("ctb")), person("Christos T.", "Nakas", role = c("ctb"), email = "cnakas@uth.gr"))
 Description: Several statistical test functions as well as a function for exploratory data analysis to investigate classifiers allocating individuals to one of three disjoint and ordered classes. In a single classifier assessment the discriminatory power is compared to classification by chance. In a comparison of two classifiers the null hypothesis corresponds to equal discriminatory power of the two classifiers.
 Depends:

trinROC/R/boot.test.R

@@ -62,7 +62,7 @@
 #' @export
 #' @references Nakas, C. T. and C. T. Yiannoutsos (2004). Ordered multiple-class
 #'  ROC analysis with continuous measurements. \emph{Statistics in
-#'  Medicine} \bold{23}(22), 3437–3449.
+#'  Medicine}, \bold{23}(22), 3437–3449.
 #' @examples
 #' data(cancer)
 #' data(krebs)

trinROC/R/roc.eda.R

@@ -211,7 +211,7 @@ roc.eda <- function(x, y, z, dat = NULL, type = c("empirical", "trinormal"),
       histROC <- ggplot(data, aes_string(x = "value", colour="trueClass", fill="trueClass")) +
         geom_histogram(aes_string(y ="..density.."), binwidth=(max(data$value)-min(data$value))/15,
                        position = "dodge", alpha=0.7, show.legend = FALSE) +
-        scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = FALSE) +
+        scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = "none") +
         scale_fill_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), name = "Class") +
         labs(y="Count",x=paste(Marker, "measurements")) +
         #facet_grid(. ~ trueClass, scales = "free") +
@@ -222,7 +222,7 @@ roc.eda <- function(x, y, z, dat = NULL, type = c("empirical", "trinormal"),
       histROC <- ggplot(data, aes_string(x = "value", colour="trueClass", fill="trueClass")) +
         geom_histogram(aes_string(y = "..density.."),binwidth=(max(data$value)-min(data$value))/30,
                        show.legend = FALSE) +
-        scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = FALSE) +
+        scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = "none") +
         scale_fill_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), name="Class") +
         labs(y="Density",x=paste(Marker, "measurements")) +
         facet_grid(. ~ trueClass, scales = "free") +
@@ -233,14 +233,14 @@ roc.eda <- function(x, y, z, dat = NULL, type = c("empirical", "trinormal"),
     # boxplots:
     meas.overview <- ggplot(data, aes_string(y="value", x="trueClass", fill = "trueClass")) +
       stat_boxplot(geom ='errorbar') + geom_boxplot() +
-      scale_fill_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = FALSE) +
+      scale_fill_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = "none") +
       coord_flip() + labs(x="", y=paste(Marker, "measurements"))
 
     } else {
     # scattter plot of data:
     meas.overview <- ggplot(data, aes_string(y="value", x="trueClass", color = "trueClass")) +
       geom_jitter(width = 0.25) +
-      scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = FALSE) +
+      scale_colour_manual(values=c("#79AB67", "#6EA3D0", "#D68898"), guide = "none") +
         coord_flip() + labs(x="", y=paste(Marker, "measurements"))
     }
 

trinROC/R/trinROC.R

@@ -3,5 +3,5 @@
 #'   estimates by setting \code{options(trinROC.MLE = FALSE)}
 #' @references Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2019).
 #'   Inference in ROC surface analysis via a trinormal model-based testing approach.
-#'   \emph{Stat}, accepted.
+#'   \emph{Stat}, \bold{8}(1), e249.
 "_PACKAGE"

trinROC/R/trinROC.test.R

@@ -71,7 +71,7 @@
 #'   geom_density geom_histogram geom_jitter labs scale_colour_manual
 #'   scale_fill_manual stat_boxplot aes_string
 #' @param conf.level confidence level of the interval. A numeric value between (0,1)
-#'   yielding the significance level \eqn{\alpha=1-\code{conf.level}}.
+#'   yielding the significance level \eqn{\alpha=1-}\code{conf.level}.
 #'
 #' @return A list of class \code{"htest"} containing the following components:
 #'   \item{statistic}{the value of the chi-squared statistic.}
@@ -89,9 +89,9 @@
 #'   \item{Summary}{a data frame representing the number of NA's as well as
 #'   the means and the standard deviations per class.}
 #'
-#' @references Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2018).
+#' @references Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2019).
 #'   Inference in ROC surface analysis via a trinormal model-based testing approach.
-#'   \emph{Submitted}.
+#'   \emph{Stat}, \bold{8}(1), e249.
 #' @seealso \code{\link{trinVUS.test}}, \code{\link{boot.test}}.
 #'
 #' @export

trinROC/R/trinVUS.test.R

@@ -82,7 +82,7 @@
 #'
 #' @references Xiong, C., van Belle, G.,  Miller, J. P.,  Yan, Y.,  Gao, F., Yu, K., and Morris, J. C. (2007). A parametric comparison
 #'   of diagnostic accuracy with three ordinal diagnostic groups.
-#'   \emph{Biometrical Journal}, \bold{49}(5), 682–693. \url{https://doi.org/10.1002/bimj.200610359}.
+#'   \emph{Biometrical Journal}, \bold{49}(5), 682–693. \doi{10.1002/bimj.200610359}.
 #' @examples
 #' data(cancer)
 #' data(krebs)

trinROC/man/boot.test.Rd

@@ -111,7 +111,7 @@ z2 <- with(krebs, krebs[trueClass=="diseased", 5])
 \references{
 Nakas, C. T. and C. T. Yiannoutsos (2004). Ordered multiple-class
  ROC analysis with continuous measurements. \emph{Statistics in
- Medicine} \bold{23}(22), 3437–3449.
+ Medicine}, \bold{23}(22), 3437–3449.
 }
 \seealso{
 \code{\link{trinROC.test}}, \code{\link{trinVUS.test}}.

trinROC/man/trinROC-package.Rd

@@ -16,7 +16,7 @@ See \code{vignette("Overview", package = "trinROC")} for an overview of the pack
 \references{
 Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2019).
   Inference in ROC surface analysis via a trinormal model-based testing approach.
-  \emph{Stat}, accepted.
+  \emph{Stat}, \bold{8}(1), e249.
 }
 \seealso{
 Useful links:

trinROC/man/trinROC.test.Rd

@@ -36,7 +36,7 @@ If \code{TRUE}, each class must have equal sample size for both
 classifiers.}
 
 \item{conf.level}{confidence level of the interval. A numeric value between (0,1)
-yielding the significance level \eqn{\alpha=1-\code{conf.level}}.}
+yielding the significance level \eqn{\alpha=1-}\code{conf.level}.}
 }
 \value{
 A list of class \code{"htest"} containing the following components:
@@ -121,9 +121,9 @@ z2 <- with(cancer, cancer[trueClass=="diseased", 5])
 trinROC.test(x1, y1, z1, x2, y2, z2, paired = TRUE)
 }
 \references{
-Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2018).
+Noll, S., Furrer, R., Reiser, B. and Nakas, C. T. (2019).
   Inference in ROC surface analysis via a trinormal model-based testing approach.
-  \emph{Submitted}.
+  \emph{Stat}, \bold{8}(1), e249.
 }
 \seealso{
 \code{\link{trinVUS.test}}, \code{\link{boot.test}}.

trinROC/man/trinVUS.test.Rd

@@ -108,7 +108,7 @@ Xiong, C., Van Belle, G.  Miller J. P., Morris, J. C. (2006). Measuring and esti
 
 Xiong, C., van Belle, G.,  Miller, J. P.,  Yan, Y.,  Gao, F., Yu, K., and Morris, J. C. (2007). A parametric comparison
   of diagnostic accuracy with three ordinal diagnostic groups.
-  \emph{Biometrical Journal}, \bold{49}(5), 682–693. \url{https://doi.org/10.1002/bimj.200610359}.
+  \emph{Biometrical Journal}, \bold{49}(5), 682–693. \doi{10.1002/bimj.200610359}.
 }
 \seealso{
 \code{\link{trinROC.test}}, \code{\link{boot.test}}.

trinROC/tests/Examples/trinROC-Ex.Rout.save

 
-R Under development (unstable) (2020-12-04 r79564) -- "Unsuffered Consequences"
-Copyright (C) 2020 The R Foundation for Statistical Computing
+R Under development (unstable) (2021-05-23 r80364) -- "Unsuffered Consequences"
+Copyright (C) 2021 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
 
 R is free software and comes with ABSOLUTELY NO WARRANTY.
@@ -535,7 +535,7 @@ VUS of Classifier 1 VUS of Classifier 2
 > cleanEx()
 > options(digits = 7L)
 > base::cat("Time elapsed: ", proc.time() - base::get("ptime", pos = 'CheckExEnv'),"\n")
-Time elapsed:  5.28 0.092 5.748 0.004 0 
+Time elapsed:  5.689 0.105 6.06 0.001 0.001 
 > grDevices::dev.off()
 null device 
           1 

trinROC/tests/testthat.Rout.save

 
-R Under development (unstable) (2020-12-04 r79564) -- "Unsuffered Consequences"
-Copyright (C) 2020 The R Foundation for Statistical Computing
+R Under development (unstable) (2021-05-23 r80364) -- "Unsuffered Consequences"
+Copyright (C) 2021 The R Foundation for Statistical Computing
 Platform: x86_64-pc-linux-gnu (64-bit)
 
 R is free software and comes with ABSOLUTELY NO WARRANTY.
@@ -53,9 +53,8 @@ Parameters:
  	a	b	c	d	
  	0.6959	-0.2546	0.4462	0.1083	
 --------------------------------------------------------------------- 
-══ testthat results  ═══════════════════════════════════════════════════════════
-[ OK: 43 | SKIPPED: 0 | WARNINGS: 0 | FAILED: 0 ]
+[ FAIL 0 | WARN 0 | SKIP 0 | PASS 43 ]
 > 
 > proc.time()
    user  system elapsed 
-  4.768   0.144   4.905 
+  5.297   0.111   5.399 

trinROC/vignettes/trinROC_vignette.tex

+% Options for packages loaded elsewhere
+\PassOptionsToPackage{unicode}{hyperref}
+\PassOptionsToPackage{hyphens}{url}
+%
+\documentclass[
+]{article}
+\usepackage{lmodern}
+\usepackage{amssymb,amsmath}
+\usepackage{ifxetex,ifluatex}
+\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
+  \usepackage[T1]{fontenc}
+  \usepackage[utf8]{inputenc}
+  \usepackage{textcomp} % provide euro and other symbols
+\else % if luatex or xetex
+  \usepackage{unicode-math}
+  \defaultfontfeatures{Scale=MatchLowercase}
+  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
+\fi
+% Use upquote if available, for straight quotes in verbatim environments
+\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
+\IfFileExists{microtype.sty}{% use microtype if available
+  \usepackage[]{microtype}
+  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
+}{}
+\makeatletter
+\@ifundefined{KOMAClassName}{% if non-KOMA class
+  \IfFileExists{parskip.sty}{%
+    \usepackage{parskip}
+  }{% else
+    \setlength{\parindent}{0pt}
+    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
+}{% if KOMA class
+  \KOMAoptions{parskip=half}}
+\makeatother
+\usepackage{xcolor}
+\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
+\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
+\hypersetup{
+  pdftitle={Overview of the Package trinROC},
+  pdfauthor={Samuel Noll, Reinhard Furrer, Benjamin Reiser and Christos T. Nakas},
+  hidelinks,
+  pdfcreator={LaTeX via pandoc}}
+\urlstyle{same} % disable monospaced font for URLs
+\usepackage[margin=1in]{geometry}
+\usepackage{color}
+\usepackage{fancyvrb}
+\newcommand{\VerbBar}{|}
+\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
+\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
+% Add ',fontsize=\small' for more characters per line
+\usepackage{framed}
+\definecolor{shadecolor}{RGB}{248,248,248}
+\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
+\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
+\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
+\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
+\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
+\newcommand{\BuiltInTok}[1]{#1}
+\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
+\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
+\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
+\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
+\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
+\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
+\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
+\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
+\newcommand{\ExtensionTok}[1]{#1}
+\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
+\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\ImportTok}[1]{#1}
+\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
+\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
+\newcommand{\NormalTok}[1]{#1}
+\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
+\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
+\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
+\newcommand{\RegionMarkerTok}[1]{#1}
+\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
+\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
+\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
+\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
+\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
+\usepackage{graphicx}
+\makeatletter
+\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
+\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
+\makeatother
+% Scale images if necessary, so that they will not overflow the page
+% margins by default, and it is still possible to overwrite the defaults
+% using explicit options in \includegraphics[width, height, ...]{}
+\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
+% Set default figure placement to htbp
+\makeatletter
+\def\fps@figure{htbp}
+\makeatother
+\setlength{\emergencystretch}{3em} % prevent overfull lines
+\providecommand{\tightlist}{%
+  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
+\setcounter{secnumdepth}{-\maxdimen} % remove section numbering
+\ifluatex
+  \usepackage{selnolig}  % disable illegal ligatures
+\fi
+\newlength{\cslhangindent}
+\setlength{\cslhangindent}{1.5em}
+\newlength{\csllabelwidth}
+\setlength{\csllabelwidth}{3em}
+\newenvironment{CSLReferences}[3] % #1 hanging-ident, #2 entry spacing
+ {% don't indent paragraphs
+  \setlength{\parindent}{0pt}
+  % turn on hanging indent if param 1 is 1
+  \ifodd #1 \everypar{\setlength{\hangindent}{\cslhangindent}}\ignorespaces\fi
+  % set entry spacing
+  \ifnum #2 > 0
+  \setlength{\parskip}{#2\baselineskip}
+  \fi
+ }%
+ {}
+\usepackage{calc} % for \widthof, \maxof
+\newcommand{\CSLBlock}[1]{#1\hfill\break}
+\newcommand{\CSLLeftMargin}[1]{\parbox[t]{\maxof{\widthof{#1}}{\csllabelwidth}}{#1}}
+\newcommand{\CSLRightInline}[1]{\parbox[t]{\linewidth}{#1}}
+\newcommand{\CSLIndent}[1]{\hspace{\cslhangindent}#1}
+
+\title{Overview of the Package \texttt{trinROC}}
+\author{Samuel Noll, Reinhard Furrer, Benjamin Reiser and Christos T.
+Nakas}
+\date{2021-01-04}
+
+\begin{document}
+\maketitle
+
+The package \texttt{trinROC} helps to assess three-class Receiver
+Operating Characteristic (ROC) type data. It provides functions for
+three statistical tests as described in Noll et al. (2019) along with
+functions for Exploratory Data Analysis (EDA) and visualization.
+
+We assume that the reader has some background in ROC analysis as well as
+some understanding of the terminology associated with Area Under the ROC
+Curve (AUC) and Volume Under the ROC Surface (VUS) indices. See Nakas
+(2014) or Noll et al. (2019) for a concise overview.
+
+This vignette consists of the following parts:
+
+\begin{itemize}
+\tightlist
+\item
+  Short background of the tests
+\item
+  Testing and comparing markers
+\item
+  Calculating empirical power curves
+\item
+  Additional functionality of the package
+\end{itemize}
+
+The package also contains two small data sets \texttt{cancer} and
+\texttt{krebs} mimicking results of clinical studies. The datasets are
+fairly small and used in this vignette (and in the examples of the help
+files) to illustrate the functionality of the functions.
+
+\hypertarget{short-background-of-the-tests}{%
+\section{Short background of the
+tests}\label{short-background-of-the-tests}}
+
+We assume a three-class setting, where one or more classifier yields
+measurements \(X = x\) on a continuous scale for the groups of healthy
+(\(D^-\)), intermediate (\(D^0\)) and diseased (\(D^+\)) individuals. By
+convention, larger values of \(x\) represent a more severe status of the
+disease. The package \texttt{trinROC} provides statistical tests to
+assess the discriminatory power of such classifiers. These tests are:
+
+\begin{itemize}
+\tightlist
+\item
+  Trinormal based ROC test (\texttt{trinROC.test}), developed by Noll et
+  al. (2019);
+\item
+  Trinormal VUS test (\texttt{trinVUS.test}), developed by Xiong et al.
+  (2007);
+\item
+  Bootstrap test, (\texttt{boot.test}), developed by Nakas and
+  Yiannoutsos (2004).
+\end{itemize}
+
+In this document, we refer to the tests by the corresponding R function
+name. All tests can assess a single classifier, as well as compare two
+paired or unpaired classifiers.
+
+As their names suggest, the underlying testing approach is different and
+either based on VUS or on ROC. Hence, their null hypotheses differ as
+well, as illustrated now.
+
+\hypertarget{vus-based-statistical-tests}{%
+\subsection{VUS based statistical
+tests}\label{vus-based-statistical-tests}}
+
+Given two classifiers, \texttt{boot.test} and \texttt{trinVUS.test} are
+based on the null hypothesis \(VUS_1 = VUS_2\) with the \(Z\)-statistic
+
+\begin{align*}
+Z =  \frac{\widehat{VUS}_1 - \widehat{VUS}_2}{\sqrt{\widehat{Var}(\widehat{VUS}_1) + \widehat{Var}(\widehat{VUS}_2) - 2 \widehat{Cov}(\widehat{VUS}_1,\widehat{VUS}_2)}}.
+\end{align*}
+
+If the data of the two classifiers is unpaired, the term
+\(Cov(VUS_1,VUS_2)\) is zero. If a single classifier is investigated,
+the null hypothesis is \(VUS_1 = 1/6\) with the \(Z\)-statistic
+
+\begin{align*}
+Z =  \frac{\widehat{VUS}_1 - 1/6}{\sqrt{\widehat{Var}(\widehat{VUS}_1)}},
+\end{align*}
+
+which is equivalent to compared the VUS of the classifier to the volume
+of an uninformative classifier. Details about the estimators are given
+in the aforementioned papers.
+
+\hypertarget{the-trinormal-based-roc-test}{%
+\subsection{The trinormal based ROC
+test}\label{the-trinormal-based-roc-test}}
+
+In the trinormal model, the ROC surface is given by
+
+\begin{align*}
+ROC(t_-,t_+) = \Phi \left(\frac{\Phi^{-1} (1-t_+) + d}{ c} \right) - \Phi \left(\frac{\Phi^{-1} (t_-)+b}{a} \right),
+\end{align*}
+
+where \(\Phi\) is the cdf of the standard normal distribution and \(a\),
+\(b\), \(c\) and \(d\) are functions of the means and standard
+deviations of the three groups:
+
+\begin{align*} 
+a = \frac{{\sigma}_0}{{\sigma}_-}, \qquad b = \frac{ {\mu}_- - {\mu}_0}{{\sigma}_-}, \qquad 
+c = \frac{{\sigma}_0}{{\sigma}_+}, \qquad d =  \frac{ {\mu}_+ - {\mu}_0}{{\sigma}_+}.
+\end{align*}
+
+Given two classifiers, \texttt{trinROC.test} investigates the shape of
+the two ROC surfaces. Hence, the resulting null hypothesis is
+\(a_1=a_2\), \(b_1=b_2\), \(c_1=c_2\) and \(d_1=d_2\), i.e., the the
+surfaces have the same shape. Under the null hypothesis, the test
+statistic
+
+\begin{align*} 
+ \chi^2 =  
+ \begin{pmatrix} \widehat{a}_1 - \widehat{a}_2 &\widehat{b}_1 -\widehat{b}_2 & \widehat{c}_1-\widehat{c}_2 & \widehat{d}_1-\widehat{d}_2 \end{pmatrix}
+ { \widehat{\boldsymbol{W}}}^{-1}
+   \begin{pmatrix} \widehat{a}_1 - \widehat{a}_2 \\
+    \widehat{b}_1 - \widehat{b}_2 \\ \widehat{c}_1-\widehat{c}_2 \\ \widehat{d}_1- \widehat{d}_2 \end{pmatrix},
+\end{align*}
+
+is distributed approximately as a chi-squared random variables with four
+degrees of freedom, where \(\widehat{\boldsymbol{W}}\) contains the
+corresponding estimated variances and covariances. The test rejects if
+\(\chi^2 > \chi_{\alpha}^2\), i.e., if the test statistics exceeds the
+chi-squared quantile with four degrees of freedom of a pre-defined
+confidence level \(\alpha\).
+
+When the data is unpaired, \(\widehat{a}_1\), \(\widehat{b}_1\),
+\(\widehat{c}_1\), \(\widehat{d}_1\) are independent from
+\(\widehat{a}_2\), \(\widehat{b}_2\), \(\widehat{c}_2\),
+\(\widehat{d}_2\), respectively, and hence every combination of
+covariances between them is zero.
+
+It is also possible to investigate a single classifier with the above
+method. Instead of an existing second classifier, we compare the
+estimates \(\widehat{a}_1\) , \(\widehat{b}_1\), \(\widehat{c}_1\) and
+\(\widehat{d}_1\) of a single marker with those from an artificial
+marker. If we want to detect whether a single marker is significantly
+better in allocating individuals to the three classes than a random
+allocation function we would set the parameters of our null hypothesis
+\(a_{Ho} = 1= c_{Ho}\), as we assume equal spread in the classes and
+\(b_{Ho} = 0 = d_{Ho}\), as we impose equal means. This yields the null
+hypothesis \(a_1=1\), \(b_1=0\), \(c_1=1\), \(d_1=0\), which leads to
+the chi-squared test
+
+\begin{align*} 
+{\chi}^2 = & 
+ \begin{pmatrix} \widehat{a}_1 -1 & \widehat{b}_1 & \widehat{c}_1 -1 & \widehat{d}_1 \end{pmatrix}
+  \widehat{\boldsymbol{W}}^{-1}
+   \begin{pmatrix} \widehat{a}_1 -1 \\ \widehat{b}_1 \\ \widehat{c}_1 - 1 \\ \widehat{d}_1 \end{pmatrix}.
+\end{align*}
+
+\hypertarget{testing-and-comparing-markers}{%
+\section{Testing and comparing
+markers}\label{testing-and-comparing-markers}}
+
+We now illustrate the use of the test functions with the artificial
+dataset \texttt{cancer}.
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\FunctionTok{data}\NormalTok{(cancer)}
+\FunctionTok{str}\NormalTok{(cancer)}
+\CommentTok{\#\textgreater{} \textquotesingle{}data.frame\textquotesingle{}:    100 obs. of  10 variables:}
+\CommentTok{\#\textgreater{}  $ trueClass: Factor w/ 3 levels "healthy","intermediate",..: 3 3 3 3 3 3 3 3 3 3 ...}
+\CommentTok{\#\textgreater{}  $ Class1   : num  1.42 1.69 1.08 1.79 1.46 0.1 1.91 2.57 1 1.87 ...}
+\CommentTok{\#\textgreater{}  $ Class2   : num  2.31 2.36 1.72 2.34 1.97 0.74 2.82 2.71 1.67 2.51 ...}
+\CommentTok{\#\textgreater{}  $ Class3   : num  {-}4647 {-}3694 {-}5119 {-}6352 {-}11509 ...}
+\CommentTok{\#\textgreater{}  $ Class4   : num  {-}1622958 {-}1591840 {-}2299714 {-}2050907 {-}2629864 ...}
+\CommentTok{\#\textgreater{}  $ Class5   : num  {-}476 {-}473 {-}618 {-}564 {-}901 ...}
+\CommentTok{\#\textgreater{}  $ Class6   : num  12.36 8.81 9.15 5.52 11.85 ...}
+\CommentTok{\#\textgreater{}  $ Class7   : num  2.48 4.38 2.3 2.41 4.68 1.34 3.18 2.35 1.83 2.83 ...}
+\CommentTok{\#\textgreater{}  $ Class8   : num  3.02 2.98 2.94 3.01 2.93 2.95 3.02 3.03 2.93 3.06 ...}
+\CommentTok{\#\textgreater{}  $ Class9   : num  {-}0.75 {-}0.86 {-}1.31 {-}0.27 {-}1.71 {-}1.39 {-}0.46 {-}1.06 {-}2.19 {-}0.91 ...}
+\end{Highlighting}
+\end{Shaded}
+
+The first column is a factor indicating the (true) class membership of
+each individual. The three levels have to be ordered according to
+heaviness of disease, i.e., healthy, intermediate and diseased (nor the
+names nor the sorting of the elements plays a role). The other columns
+contain the measurements yielded by Classifier 1 to 9. Further we note
+that some columns where multiplied by \(-1\) in order to fulfill the
+convention that more diseased individuals have (in general) higher
+measurements.
+
+\hypertarget{single-marker-assessment}{%
+\subsection{Single marker assessment}\label{single-marker-assessment}}
+
+For illustration, let us assess \texttt{Class2} of the data set
+\texttt{cancer} using the function \texttt{trinROC.test}.
+
+\begin{Shaded}
+\begin{Highlighting}[]
+\NormalTok{out }\OtherTok{\textless{}{-}} \FunctionTok{trinROC.test}\NormalTok{(}\AttributeTok{dat =}\NormalTok{ cancer[,}\FunctionTok{c}\NormalTok{(}\StringTok{"trueClass"}\NormalTok{,}\StringTok{"Class2"}\NormalTok{)])}
+\NormalTok{out}
+\CommentTok{\#\textgreater{} }
+\CommentTok{\#\textgreater{}  Trinormal based ROC test for single classifier assessment}
+\CommentTok{\#\textgreater{} }
+\CommentTok{\#\textgreater{} data:  healthy  intermediate  diseased  of  Class2}