eu.project.ttc.resources.EvalTrace Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of termsuite-core Show documentation
A Java UIMA-based toolbox for multilingual and efficient terminology extraction an multilingual term alignment
There is a newer version: 3.0.10
/*******************************************************************************
 * Copyright 2015 - CNRS (Centre National de Recherche Scientifique)
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 *******************************************************************************/
package eu.project.ttc.resources;

import java.net.URI;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;

import org.apache.uima.resource.DataResource;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.SharedResourceObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Joiner;
import com.google.common.base.MoreObjects;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

import eu.project.ttc.engines.EvalEngine;
import eu.project.ttc.resources.ReferenceTermList.RTLTerm;

/**
 * 
 * A list of evaluation {@link RecPoint} for precision/recall metrics against against 
 * a reference list. This resource is used and filled by {@link EvalEngine}.
 * 
 * Each {@link RecPoint}  holds the number of true positive (tp) extracted 
 * terms at a given index (where index is the top n extracted terms) aginst the
 * reference list.
 * 
 * @author Damien Cram
 *
 */
public class EvalTrace implements SharedResourceObject {
	private static final Logger LOGGER = LoggerFactory.getLogger(EvalTrace.class);
	
	private static Map evalTraces = Maps.newConcurrentMap();
	
	/**
	 * 
	 * 
	 * 
	 * @author Damien Cram
	 *
	 */
	public class RecPoint {
		private static final String PRECISION = "precision";
		private static final String RECALL = "recall";
		List lcTermsFound;
		int rank;
		int tp;
		RecPoint(int rank, int tp, List lcTermsFound) {
			super();
			this.rank = rank;
			this.tp = tp;
			this.lcTermsFound = lcTermsFound;
		}
		public int getTp() {
			return tp;
		}
		public int getRank() {
			return this.rank;
		}
		public double getFScore(double beta) {
			if(getPrecision()*getRecall() == 0d)
				return 0d;
			else
				return ((1 + beta*beta)*getPrecision()*getRecall())/(beta*beta*(getPrecision()+getRecall()));
		}
		public double getPrecision() {
			return ((float)this.tp) / this.rank;
		}
		public double getRecall() {
			return ((float)this.tp) / EvalTrace.this.rtlSize;
		}
		public double getValue(String pName) {
			switch (pName) {
			case PRECISION:
				return getPrecision();
			case RECALL:
				return getRecall();
			default:
				throw  new IllegalArgumentException("Unkown property name for RecPoint: " + pName);
			}
		}

		public List getLcTermsFound() {
			return lcTermsFound;
		}
		
		@Override
		public String toString() {
			return String.format("%d: %d", rank, tp);
		}
	} /* End RecPoint */
	
	private int rtlSize;
	public void setRtlSize(int refRtlSize) {
		this.rtlSize = refRtlSize;
	}
	
	private SortedMap points = Maps.newTreeMap();
	
	private String name;
	

	public void trace(int index, int tp, List foundTerms) {
		points.put(index, new RecPoint(index, tp, foundTerms));
	}
	
	public double getMaxRecall() {
		return points.isEmpty() ? 1d : getLast().getRecall();
	}
	
	public RecPoint getLast() {
		return points.get(points.lastKey());
	}
	
	@Override
	public String toString() {
		return MoreObjects.toStringHelper(this)
				.addValue(this.name)
				.add("rtlSize", this.rtlSize)
				.add("points", Joiner.on("|").join(this.points.values()))
				.toString();
	}
	/**
	 * Get the first {@link RecPoint} whose recall exceeds the parameter
	 * precisionThreshorld.
	 * 
	 * @param recallThreshorld
	 * @return
	 */
	public RecPoint getAtRecall(double recallThreshorld) {
		RecPoint last = null;
		for(java.util.Map.Entry e:points.entrySet()) {
			if(e.getValue().getRecall() >= recallThreshorld)
				return e.getValue();
			last = e.getValue();
		}
		return last;
	}

	/**
	 * Get the {@link RecPoint} for rank number of extracted indexes.
	 * 
	 * @param rank
	 * @return
	 */
	public RecPoint getAtRank(int rank) {
		return points.get(rank);
	}

	public Collection getIndexes() {
		return points.keySet();
	}

	public Collection getValues(String pName) {
		List vals = Lists.newArrayList();
		for(RecPoint p:points.values())
			vals.add(p.getValue(pName));
		return vals;
	}

	public Collection getPoints() {
		return points.values();
	}
	public Collection getChartAxisPoints(boolean takeLastPoint) {
		List axisPoints = Lists.newArrayList();
		List accu = Lists.newArrayList();
		RecPoint last = null;
		for(RecPoint p:points.values()) {
			last = p;
			accu.addAll(p.getLcTermsFound());
			if(EvalEngine.CHART_AXIS_POINTS.contains(p.getRank())) {
				axisPoints.add(new RecPoint(p.getRank(), p.getTp(), accu));
				accu = Lists.newArrayList();
			}
		}
		if(takeLastPoint)
			axisPoints.add(new RecPoint(last.getRank(), last.getTp(), accu));
		return axisPoints;
	}

	
	@Override
	public void load(DataResource aData) throws ResourceInitializationException {
		URI uri = aData.getUri();
		String tName = uri.toString();
		LOGGER.debug("Registering trace {}", tName);
		if(tName == null)
			throw new ResourceInitializationException("Could not initialize an evaluation with a null name.", new Object[]{tName});
		else if(EvalTrace.evalTraces.containsKey(tName))
			throw new ResourceInitializationException("A trace named "+tName+" already exists", new Object[]{tName});
		else {
			this.name = tName;
			EvalTrace.evalTraces.put(tName, this);
		}
	}

	/**
	 * Remove all evaluation traces.
	 */
	public static void clear() {
		evalTraces.clear();
	}
	
	public static EvalTrace get(String traceName) {
		return evalTraces.get(traceName);
	}
	
	
	
	public String getName() {
		return name;
	}

	/**
	 * Creates a clone of this trace with only points that are 
	 * meant to be displayed on a chart (these points are {@link EvalEngine#CHART_AXIS_POINTS})
	 * 
	 * @return
	 */
	public EvalTrace toDisplayable() {
		EvalTrace displayable = new EvalTrace();
		displayable.rtlSize = this.rtlSize;
		displayable.name = this.name;
		for(RecPoint p:this.getChartAxisPoints(true)) 
			displayable.points.put(p.getRank(), p);
		return displayable;
	}
	
	/**
	 * Filters this eval trace and keeps only the parameter indexes.
	 */
	public void keepIndexes(Collection indexes) {
		Iterator> it = points.entrySet().iterator();
		while(it.hasNext()) {
			Map.Entry e = it.next();
			if(!indexes.contains(e.getKey()))
				it.remove();
		}
	}
}