# main script generating results for the COLING paper

source("aggregators.R")
source("evaluation.R")

# note that all critical columns must be of type character!!
rte <- read.csv("RTE-annotations.csv", colClasses="character")
psd <- read.csv("PSD-annotations.csv", colClasses="character")
qda <- read.csv("QDA-annotations.csv", colClasses="character")

rte.gold <- ReadGold(rte)
psd.gold <- ReadGold(psd)
qda.gold <- ReadGold(qda)

aggregators <- c(SPR, COM, INV, DIFF, RAT, AGR, ORA)
names(aggregators) <- c("SPR", "COM", "INV", "DIFF", "RAT", "AGR", "ORA")

print("Processing the RTE dataset")
rte.oa <- sapply(aggregators, 
                       function(aggregator){ObservedAgreement(aggregator(rte),
                                                              rte.gold)})
rte.result <- data.frame(OA=rte.oa)
print(rte.result)

print(paste("Observed Agreement for AGR with prior:", 
            ObservedAgreement(AGR.PRIOR(rte), rte.gold)))
print("Iterating AGR for RTE")
print(ObservedAgreement(AGR.ITER(rte), rte.gold))


print("Processing the PSD dataset")
psd.oa <- sapply(aggregators, 
                 function(aggregator){ObservedAgreement(aggregator(psd),
                                                        psd.gold)})
psd.result <- data.frame(OA=psd.oa)
print(psd.result)

print(paste("Observed Agreement for AGR with prior:", 
            ObservedAgreement(AGR.PRIOR(psd), psd.gold)))
print("Iterating AGR for PSD")
print(ObservedAgreement(AGR.ITER(psd), psd.gold))


print("Processing the QDA dataset")
qda.oa <- sapply(aggregators, 
                 function(aggregator){ObservedAgreement(aggregator(qda),
                                                        qda.gold)})
qda.result <- data.frame(OA=qda.oa)
print(qda.result)

print(paste("Observed Agreement for AGR with prior:", 
            ObservedAgreement(AGR.PRIOR(qda), qda.gold)))
print("Iterating AGR for QDA")
print(ObservedAgreement(AGR.ITER(qda), qda.gold))