001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.model.asm;
031
032import java.util.ArrayList;
033import java.util.List;
034
035import org.openimaj.citation.annotation.Reference;
036import org.openimaj.citation.annotation.ReferenceType;
037import org.openimaj.citation.annotation.References;
038import org.openimaj.image.Image;
039import org.openimaj.image.model.landmark.LandmarkModel;
040import org.openimaj.image.model.landmark.LandmarkModelFactory;
041import org.openimaj.math.geometry.point.Point2d;
042import org.openimaj.math.geometry.shape.PointDistributionModel;
043import org.openimaj.math.geometry.shape.PointDistributionModel.Constraint;
044import org.openimaj.math.geometry.shape.PointList;
045import org.openimaj.math.matrix.algorithm.pca.PrincipalComponentAnalysis.ComponentSelector;
046import org.openimaj.util.pair.IndependentPair;
047import org.openimaj.util.pair.ObjectFloatPair;
048
049import Jama.Matrix;
050
051/**
052 * Implementation of a basic Active Shape Model. The implementation allows different
053 * types of landmark appearance models and can work with both colour and greylevel
054 * images.
055 * 
056 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
057 * 
058 * @param <I> Concrete type of {@link Image}
059 */
060@References(references = { 
061                @Reference(
062                                author = { "Cootes, T. F.", "Taylor, C. J." }, 
063                                title = "Statistical Models of Appearance for Computer Vision", 
064                                type = ReferenceType.Unpublished,
065                                month = "October",
066                                year = "2001",
067                                url = "http://isbe.man.ac.uk/~bim/Models/app_model.ps.gz"
068                ), 
069                @Reference(
070                                type = ReferenceType.Inproceedings,
071                                author = { "T. F. Cootes", "C. J. Taylor" },
072                                title = "Active Shape Models",
073                                year = "1992",
074                                booktitle = "in Proceedings of the British Machine Vision Conference"
075                )
076})
077public class ActiveShapeModel<I extends Image<?, I>> {
078        private PointDistributionModel pdm;
079        private LandmarkModel<I>[] landmarkModels;
080        private int maxIter = 50;
081        private double inlierPercentage = 0.9;
082        
083        /**
084         * Construct an {@link ActiveShapeModel} from a pre-trained {@link PointDistributionModel}
085         * and set of {@link LandmarkModel}s.
086         * @param pdm the {@link PointDistributionModel}.
087         * @param landmarkModels the {@link LandmarkModel}s.
088         */
089        public ActiveShapeModel(PointDistributionModel pdm, LandmarkModel<I>[] landmarkModels) {
090                this.pdm = pdm;
091                this.landmarkModels = landmarkModels;
092        }
093        
094        /**
095         * Train a new {@link ActiveShapeModel} using the given data
096         * and parameters.
097         * 
098         * @param <I> The concrete image type.
099         * @param selector the selector for choosing the number of principal components / modes of the model.
100         * @param data the data to train the model from
101         * @param constraint the constraint to apply to restrict the model to plausible shapes.
102         * @param factory the {@link LandmarkModelFactory} for learning local appearance models
103         * @return a newly trained {@link ActiveShapeModel}.
104         */
105        public static <I extends Image<?, I>> ActiveShapeModel<I> trainModel(ComponentSelector selector, List<IndependentPair<PointList, I>> data, Constraint constraint, LandmarkModelFactory<I> factory) {
106                int nPoints = data.get(0).firstObject().size();
107                
108                @SuppressWarnings("unchecked")
109                LandmarkModel<I>[] ppms = new LandmarkModel[nPoints];
110                
111                for (int i=0; i<data.size(); i++) {
112                        for (int j=0; j<nPoints; j++) {
113                                if (ppms[j] == null) {
114                                        ppms[j] = factory.createLandmarkModel();
115                                }
116                        
117                                PointList pl = data.get(i).firstObject();
118                                
119                                ppms[j].updateModel(data.get(i).secondObject(), pl.get(j), pl);
120                        }
121                }
122                
123                List<PointList> pls = new ArrayList<PointList>();
124                for (IndependentPair<PointList, I> i : data)
125                        pls.add(i.firstObject());
126                
127                PointDistributionModel pdm = new PointDistributionModel(constraint, pls);
128                pdm.setNumComponents(selector);
129                
130                return new ActiveShapeModel<I>(pdm, ppms);
131        }
132        
133        /**
134         * Class to hold the response of a single iteration
135         * of model fitting.
136         * 
137         * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
138         *
139         */
140        public static class IterationResult {
141                /**
142                 * The percentage of points that moved less than 50% of there allowed distance
143                 */
144                public double fit;
145                /**
146                 * The updated shape in image coordinates
147                 */
148                public PointList shape;
149                /**
150                 * The model pose from model coordinates to image coordinates
151                 */
152                public Matrix pose;
153                /**
154                 * The model weight parameters
155                 */
156                public double [] parameters;
157
158                protected IterationResult(Matrix pose, PointList shape, double fit, double [] parameters) {
159                        this.pose = pose;
160                        this.shape = shape;
161                        this.fit = fit;
162                        this.parameters = parameters;
163                }
164        }
165        
166        /**
167         * Perform a single iteration of model fitting.
168         * 
169         * @param image the image to fit to
170         * @param currentShape the starting shape in image coordinates
171         * @return the updated shape and parameters
172         */
173        public IterationResult performIteration(I image, PointList currentShape) {
174                PointList newShape = new PointList();
175                
176                int inliers = 0;
177                int outliers = 0;
178                //compute updated points and a score based on how far they moved
179                for (int i=0; i<landmarkModels.length; i++) {
180                        ObjectFloatPair<Point2d> newBest = landmarkModels[i].updatePosition(image, currentShape.get(i), currentShape);
181                        newShape.points.add( newBest.first );
182                        
183                        float percentageFromStart = newBest.second;
184                        if (percentageFromStart < 0.5)
185                                inliers++;
186                        else
187                                outliers++;
188                }
189                double score = ((double)inliers) / ((double)(inliers + outliers));
190                
191                //find the parameters and pose that "best" model the updated points
192                IndependentPair<Matrix, double[]> newModelParams = pdm.fitModel(newShape);
193                
194                Matrix pose = newModelParams.firstObject();
195                double[] parameters = newModelParams.secondObject();
196                
197                //apply model parameters to get final shape for the iteration
198                newShape = pdm.generateNewShape(parameters).transform(pose);
199                
200                return new IterationResult(pose, newShape, score, parameters);
201        }
202        
203        /**
204         * Iteratively apply {@link #performIteration(Image, PointList)} until
205         * the maximum number of iterations is exceeded, or the number of
206         * points that moved less than 0.5 of their maximum distance in an
207         * iteration is less than the target inlier percentage.
208         * 
209         * @see #setInlierPercentage(double)
210         * @see #setMaxIterations(int)
211         * 
212         * @param image the image to fit the shape to 
213         * @param initialShape the initial shape in image coordinates
214         * @return the fitted shape and parameters
215         */
216        public IterationResult fit(I image, PointList initialShape) {
217                IterationResult ir = performIteration(image, initialShape);
218                int count = 0;
219                
220                while (ir.fit < inlierPercentage  && count < maxIter) {
221                        ir = performIteration(image, ir.shape);
222                        count++;
223                }
224                
225                return ir;
226        }
227        
228        /**
229         * @return the maxIter
230         */
231        public int getMaxIterations() {
232                return maxIter;
233        }
234
235        /**
236         * Set the maximum allowed number of iterations in fitting the model
237         * @param maxIter the maxIter to set
238         */
239        public void setMaxIterations(int maxIter) {
240                this.maxIter = maxIter;
241        }
242
243        /**
244         * @return the inlierPercentage
245         */
246        public double getInlierPercentage() {
247                return inlierPercentage;
248        }
249
250        /**
251         * Set the target percentage of the number of points that
252         * move less than 0.5 of their total possible distance within
253         * an iteration to stop fitting.
254         *  
255         * @param inlierPercentage the inlierPercentage to set
256         */
257        public void setInlierPercentage(double inlierPercentage) {
258                this.inlierPercentage = inlierPercentage;
259        }
260
261        /**
262         * @return the learnt {@link PointDistributionModel}
263         */
264        public PointDistributionModel getPDM() {
265                return pdm;
266        }
267
268        /**
269         * @return the local landmark appearance models; one for each point in the shape.
270         */
271        public LandmarkModel<I>[] getLandmarkModels() {
272                return landmarkModels;
273        }
274}