001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.model.asm; 031 032import java.util.ArrayList; 033import java.util.List; 034 035import org.openimaj.citation.annotation.Reference; 036import org.openimaj.citation.annotation.ReferenceType; 037import org.openimaj.citation.annotation.References; 038import org.openimaj.image.Image; 039import org.openimaj.image.model.landmark.LandmarkModel; 040import org.openimaj.image.model.landmark.LandmarkModelFactory; 041import org.openimaj.math.geometry.point.Point2d; 042import org.openimaj.math.geometry.shape.PointDistributionModel; 043import org.openimaj.math.geometry.shape.PointDistributionModel.Constraint; 044import org.openimaj.math.geometry.shape.PointList; 045import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.ComponentSelector; 046import org.openimaj.util.pair.IndependentPair; 047import org.openimaj.util.pair.ObjectFloatPair; 048 049import Jama.Matrix; 050 051/** 052 * Implementation of a basic Active Shape Model. The implementation allows different 053 * types of landmark appearance models and can work with both colour and greylevel 054 * images. 055 * 056 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 057 * 058 * @param <I> Concrete type of {@link Image} 059 */ 060@References(references = { 061 @Reference( 062 author = { "Cootes, T. F.", "Taylor, C. J." }, 063 title = "Statistical Models of Appearance for Computer Vision", 064 type = ReferenceType.Unpublished, 065 month = "October", 066 year = "2001", 067 url = "http://isbe.man.ac.uk/~bim/Models/app_model.ps.gz" 068 ), 069 @Reference( 070 type = ReferenceType.Inproceedings, 071 author = { "T. F. Cootes", "C. J. Taylor" }, 072 title = "Active Shape Models", 073 year = "1992", 074 booktitle = "in Proceedings of the British Machine Vision Conference" 075 ) 076}) 077public class ActiveShapeModel<I extends Image<?, I>> { 078 private PointDistributionModel pdm; 079 private LandmarkModel<I>[] landmarkModels; 080 private int maxIter = 50; 081 private double inlierPercentage = 0.9; 082 083 /** 084 * Construct an {@link ActiveShapeModel} from a pre-trained {@link PointDistributionModel} 085 * and set of {@link LandmarkModel}s. 086 * @param pdm the {@link PointDistributionModel}. 087 * @param landmarkModels the {@link LandmarkModel}s. 088 */ 089 public ActiveShapeModel(PointDistributionModel pdm, LandmarkModel<I>[] landmarkModels) { 090 this.pdm = pdm; 091 this.landmarkModels = landmarkModels; 092 } 093 094 /** 095 * Train a new {@link ActiveShapeModel} using the given data 096 * and parameters. 097 * 098 * @param <I> The concrete image type. 099 * @param selector the selector for choosing the number of principal components / modes of the model. 100 * @param data the data to train the model from 101 * @param constraint the constraint to apply to restrict the model to plausible shapes. 102 * @param factory the {@link LandmarkModelFactory} for learning local appearance models 103 * @return a newly trained {@link ActiveShapeModel}. 104 */ 105 public static <I extends Image<?, I>> ActiveShapeModel<I> trainModel(ComponentSelector selector, List<IndependentPair<PointList, I>> data, Constraint constraint, LandmarkModelFactory<I> factory) { 106 int nPoints = data.get(0).firstObject().size(); 107 108 @SuppressWarnings("unchecked") 109 LandmarkModel<I>[] ppms = new LandmarkModel[nPoints]; 110 111 for (int i=0; i<data.size(); i++) { 112 for (int j=0; j<nPoints; j++) { 113 if (ppms[j] == null) { 114 ppms[j] = factory.createLandmarkModel(); 115 } 116 117 PointList pl = data.get(i).firstObject(); 118 119 ppms[j].updateModel(data.get(i).secondObject(), pl.get(j), pl); 120 } 121 } 122 123 List<PointList> pls = new ArrayList<PointList>(); 124 for (IndependentPair<PointList, I> i : data) 125 pls.add(i.firstObject()); 126 127 PointDistributionModel pdm = new PointDistributionModel(constraint, pls); 128 pdm.setNumComponents(selector); 129 130 return new ActiveShapeModel<I>(pdm, ppms); 131 } 132 133 /** 134 * Class to hold the response of a single iteration 135 * of model fitting. 136 * 137 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 138 * 139 */ 140 public static class IterationResult { 141 /** 142 * The percentage of points that moved less than 50% of there allowed distance 143 */ 144 public double fit; 145 /** 146 * The updated shape in image coordinates 147 */ 148 public PointList shape; 149 /** 150 * The model pose from model coordinates to image coordinates 151 */ 152 public Matrix pose; 153 /** 154 * The model weight parameters 155 */ 156 public double [] parameters; 157 158 protected IterationResult(Matrix pose, PointList shape, double fit, double [] parameters) { 159 this.pose = pose; 160 this.shape = shape; 161 this.fit = fit; 162 this.parameters = parameters; 163 } 164 } 165 166 /** 167 * Perform a single iteration of model fitting. 168 * 169 * @param image the image to fit to 170 * @param currentShape the starting shape in image coordinates 171 * @return the updated shape and parameters 172 */ 173 public IterationResult performIteration(I image, PointList currentShape) { 174 PointList newShape = new PointList(); 175 176 int inliers = 0; 177 int outliers = 0; 178 //compute updated points and a score based on how far they moved 179 for (int i=0; i<landmarkModels.length; i++) { 180 ObjectFloatPair<Point2d> newBest = landmarkModels[i].updatePosition(image, currentShape.get(i), currentShape); 181 newShape.points.add( newBest.first ); 182 183 float percentageFromStart = newBest.second; 184 if (percentageFromStart < 0.5) 185 inliers++; 186 else 187 outliers++; 188 } 189 double score = ((double)inliers) / ((double)(inliers + outliers)); 190 191 //find the parameters and pose that "best" model the updated points 192 IndependentPair<Matrix, double[]> newModelParams = pdm.fitModel(newShape); 193 194 Matrix pose = newModelParams.firstObject(); 195 double[] parameters = newModelParams.secondObject(); 196 197 //apply model parameters to get final shape for the iteration 198 newShape = pdm.generateNewShape(parameters).transform(pose); 199 200 return new IterationResult(pose, newShape, score, parameters); 201 } 202 203 /** 204 * Iteratively apply {@link #performIteration(Image, PointList)} until 205 * the maximum number of iterations is exceeded, or the number of 206 * points that moved less than 0.5 of their maximum distance in an 207 * iteration is less than the target inlier percentage. 208 * 209 * @see #setInlierPercentage(double) 210 * @see #setMaxIterations(int) 211 * 212 * @param image the image to fit the shape to 213 * @param initialShape the initial shape in image coordinates 214 * @return the fitted shape and parameters 215 */ 216 public IterationResult fit(I image, PointList initialShape) { 217 IterationResult ir = performIteration(image, initialShape); 218 int count = 0; 219 220 while (ir.fit < inlierPercentage && count < maxIter) { 221 ir = performIteration(image, ir.shape); 222 count++; 223 } 224 225 return ir; 226 } 227 228 /** 229 * @return the maxIter 230 */ 231 public int getMaxIterations() { 232 return maxIter; 233 } 234 235 /** 236 * Set the maximum allowed number of iterations in fitting the model 237 * @param maxIter the maxIter to set 238 */ 239 public void setMaxIterations(int maxIter) { 240 this.maxIter = maxIter; 241 } 242 243 /** 244 * @return the inlierPercentage 245 */ 246 public double getInlierPercentage() { 247 return inlierPercentage; 248 } 249 250 /** 251 * Set the target percentage of the number of points that 252 * move less than 0.5 of their total possible distance within 253 * an iteration to stop fitting. 254 * 255 * @param inlierPercentage the inlierPercentage to set 256 */ 257 public void setInlierPercentage(double inlierPercentage) { 258 this.inlierPercentage = inlierPercentage; 259 } 260 261 /** 262 * @return the learnt {@link PointDistributionModel} 263 */ 264 public PointDistributionModel getPDM() { 265 return pdm; 266 } 267 268 /** 269 * @return the local landmark appearance models; one for each point in the shape. 270 */ 271 public LandmarkModel<I>[] getLandmarkModels() { 272 return landmarkModels; 273 } 274}