001/**
002 * Copyright (c) 2011, The University of Southampton and the individual contributors.
003 * All rights reserved.
004 *
005 * Redistribution and use in source and binary forms, with or without modification,
006 * are permitted provided that the following conditions are met:
007 *
008 *   *  Redistributions of source code must retain the above copyright notice,
009 *      this list of conditions and the following disclaimer.
010 *
011 *   *  Redistributions in binary form must reproduce the above copyright notice,
012 *      this list of conditions and the following disclaimer in the documentation
013 *      and/or other materials provided with the distribution.
014 *
015 *   *  Neither the name of the University of Southampton nor the names of its
016 *      contributors may be used to endorse or promote products derived from this
017 *      software without specific prior written permission.
018 *
019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
029 */
030package org.openimaj.image.feature.local.interest;
031
032import java.util.ArrayList;
033import java.util.List;
034
035import odk.lang.FastMath;
036
037import org.openimaj.algorithm.iterative.IterationState;
038import org.openimaj.image.FImage;
039import org.openimaj.image.processing.convolution.FImageConvolveSeparable;
040import org.openimaj.math.geometry.point.Point2d;
041import org.openimaj.math.geometry.point.Point2dImpl;
042import org.openimaj.math.matrix.PseudoInverse;
043import org.openimaj.util.function.Predicate;
044
045import Jama.Matrix;
046
047/**
048 * Refines detected corners (i.e. from {@link HarrisIPD} or other methods) to an
049 * optimised sub-pixel estimate. The method works by iteratively updating the
050 * sub-pixel position of a point based on the inverse of the local gradient
051 * auto-correlation matrix.
052 * 
053 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
054 */
055public class SubPixelCorners {
056        private static final float GRAD_X_KERNEL[] = { -1.f, 0.f, 1.f };
057        private static final float GRAD_Y_KERNEL[] = { 0.f, 0.5f, 0.f };
058
059        private Predicate<IterationState> iter;
060        private int halfWidth;
061        private int halfHeight;
062        private int zeroZoneHalfWidth = -1;
063        private int zeroZoneHalfHeight = -1;
064
065        /**
066         * Construct with the given search window size and predicate to <b>stop</b>
067         * the iteration. No zeroing of weights is performed
068         * 
069         * @param halfWidth
070         *            half the search window width (actual size is 2*halfWidth + 1,
071         *            centred on point)
072         * @param halfHeight
073         *            half the search window height (actual size is 2*halfHeight +
074         *            1, centred on point)
075         * @param iter
076         *            the predicate to stop iteration
077         */
078        public SubPixelCorners(int halfWidth, int halfHeight, Predicate<IterationState> iter) {
079                this.halfWidth = halfWidth;
080                this.halfHeight = halfHeight;
081                this.iter = iter;
082        }
083
084        /**
085         * Construct with the given search window size, zeroed window and predicate
086         * to <b>stop</b> the iteration.
087         * <p>
088         * The zeroed window is a dead region in the middle of the search zone over
089         * which the summation over gradients is not done. It is used sometimes to
090         * avoid possible singularities of the autocorrelation matrix.
091         * 
092         * @param halfWidth
093         *            half the search window width (actual size is 2*halfWidth + 1,
094         *            centred on point)
095         * @param halfHeight
096         *            half the search window height (actual size is 2*halfHeight +
097         *            1, centred on point)
098         * @param zeroZoneHalfWidth
099         *            the half-width of the zeroed region in the weighting array
100         * @param zeroZoneHalfHeight
101         *            the half-height of the zeroed region in the weighting array
102         * @param iter
103         *            the predicate to stop iteration
104         */
105        public SubPixelCorners(int halfWidth, int halfHeight, int zeroZoneHalfWidth, int zeroZoneHalfHeight,
106                        Predicate<IterationState> iter)
107        {
108                this.halfWidth = halfWidth;
109                this.halfHeight = halfHeight;
110                this.zeroZoneHalfWidth = zeroZoneHalfWidth;
111                this.zeroZoneHalfHeight = zeroZoneHalfHeight;
112                this.iter = iter;
113        }
114
115        /**
116         * Find the sub-pixel estimated position of each corner
117         * 
118         * @param src
119         *            the image
120         * @param corners
121         *            the initial corner positions
122         * @return the updated corners
123         */
124        public List<Point2dImpl> findSubPixCorners(FImage src, List<? extends Point2d> corners)
125        {
126                final List<Point2dImpl> outCorners = new ArrayList<Point2dImpl>(corners.size());
127                final int windowWidth = halfWidth * 2 + 1;
128                final int windowHeight = halfHeight * 2 + 1;
129
130                if (corners.size() == 0)
131                        return outCorners;
132
133                final FImage weights = this.buildGaussianWeights(windowWidth, windowHeight);
134                final FImage gx = new FImage(windowWidth, windowHeight);
135                final FImage gy = new FImage(windowWidth, windowHeight);
136                final float[] buffer = new float[windowWidth + 2];
137
138                // note 2px padding as conv reduces size:
139                final FImage roi = new FImage(windowWidth + 2, windowHeight + 2);
140
141                // loop for all the points
142                for (int i = 0; i < corners.size(); i++)
143                {
144                        final Point2d pt = corners.get(i);
145                        outCorners.add(this.findCornerSubPix(src, pt, roi, gx, gy, weights, buffer));
146                }
147
148                return outCorners;
149        }
150
151        /**
152         * Find the sub-pixel estimated position of a corner
153         * 
154         * @param src
155         *            the image
156         * @param corner
157         *            the initial corner position
158         * @return the updated corner position
159         */
160        public Point2dImpl findSubPixCorner(FImage src, Point2d corner)
161        {
162                final int windowWidth = halfWidth * 2 + 1;
163                final int windowHeight = halfHeight * 2 + 1;
164
165                final FImage weights = this.buildGaussianWeights(windowWidth, windowHeight);
166                final FImage gx = new FImage(windowWidth, windowHeight);
167                final FImage gy = new FImage(windowWidth, windowHeight);
168                final float[] buffer = new float[windowWidth + 2];
169
170                // note 2px padding as conv reduces size:
171                final FImage roi = new FImage(windowWidth + 2, windowHeight + 2);
172
173                return this.findCornerSubPix(src, corner, roi, gx, gy, weights, buffer);
174        }
175
176        /**
177         * Build the Gaussian weighting image
178         * 
179         * @param width
180         *            the width
181         * @param height
182         *            the height
183         * @return
184         */
185        private FImage buildGaussianWeights(int width, int height) {
186                final FImage weights = new FImage(width, height);
187                final float[] weightsX = new float[width];
188
189                double coeff = 1. / (halfWidth * halfWidth);
190                for (int i = -halfWidth, k = 0; i <= halfWidth; i++, k++)
191                {
192                        weightsX[k] = (float) Math.exp(-i * i * coeff);
193                }
194
195                float[] weightsY;
196                if (halfWidth == halfHeight) {
197                        weightsY = weightsX;
198                } else {
199                        weightsY = new float[height];
200                        coeff = 1. / (halfHeight * halfHeight);
201                        for (int i = -halfHeight, k = 0; i <= halfHeight; i++, k++)
202                        {
203                                weightsY[k] = (float) Math.exp(-i * i * coeff);
204                        }
205                }
206
207                for (int i = 0; i < height; i++) {
208                        for (int j = 0; j < width; j++) {
209                                weights.pixels[i][j] = weightsX[j] * weightsY[i];
210                        }
211                }
212
213                // if necessary, mask off the centre portion
214                if (zeroZoneHalfWidth >= 0 && zeroZoneHalfHeight >= 0 &&
215                                zeroZoneHalfWidth * 2 + 1 < width && zeroZoneHalfHeight * 2 + 1 < height)
216                {
217                        for (int i = halfHeight - zeroZoneHalfHeight; i <= halfHeight + zeroZoneHalfHeight; i++) {
218                                for (int j = halfWidth - zeroZoneHalfWidth; j <= halfWidth + zeroZoneHalfWidth; j++) {
219                                        weights.pixels[i][j] = 0;
220                                }
221                        }
222                }
223
224                return weights;
225        }
226
227        private Point2dImpl findCornerSubPix(FImage src, Point2d pt, FImage roi, FImage gx, FImage gy, FImage weights,
228                        final float[] buffer)
229        {
230                final IterationState state = new IterationState();
231                Point2dImpl current = new Point2dImpl(pt);
232
233                while (!iter.test(state)) {
234                        // get window around pt
235                        src.extractCentreSubPix(current, roi);
236
237                        // calc derivatives
238                        FImageConvolveSeparable.fastConvolve3(roi, gx, GRAD_X_KERNEL, GRAD_Y_KERNEL, buffer);
239                        FImageConvolveSeparable.fastConvolve3(roi, gy, GRAD_Y_KERNEL, GRAD_X_KERNEL, buffer);
240
241                        double a = 0, b = 0, c = 0, bb1 = 0, bb2 = 0;
242                        final int win_w = weights.width;
243                        final int win_h = weights.height;
244
245                        // process gradient
246                        for (int i = 0; i < win_h; i++)
247                        {
248                                final double py = i - halfHeight;
249
250                                for (int j = 0; j < win_w; j++)
251                                {
252                                        final double m = weights.pixels[i][j];
253                                        final double tgx = gx.pixels[i][j];
254                                        final double tgy = gy.pixels[i][j];
255                                        final double gxx = tgx * tgx * m;
256                                        final double gxy = tgx * tgy * m;
257                                        final double gyy = tgy * tgy * m;
258                                        final double px = j - halfWidth;
259
260                                        a += gxx;
261                                        b += gxy;
262                                        c += gyy;
263
264                                        bb1 += gxx * px + gxy * py;
265                                        bb2 += gxy * px + gyy * py;
266                                }
267                        }
268
269                        final Matrix m = new Matrix(new double[][] { { a, b }, { b, c } });
270                        final Matrix mInv = PseudoInverse.pseudoInverse(m);
271                        final Point2dImpl cI2 = new Point2dImpl();
272                        cI2.x = (float) (current.x + mInv.get(0, 0) * bb1 + mInv.get(0, 1) * bb2);
273                        cI2.y = (float) (current.y + mInv.get(1, 0) * bb1 + mInv.get(1, 1) * bb2);
274
275                        state.epsilon = FastMath.sqrt((cI2.x - current.x) * (cI2.x - current.x) + (cI2.y - current.y)
276                                        * (cI2.y - current.y));
277                        state.iteration++;
278                        current = cI2;
279                }
280
281                // if new point is too far from initial, it means poor convergence.
282                // return initial point as the result
283                if (Math.abs(current.x - pt.getX()) > halfWidth || Math.abs(current.y - pt.getY()) > halfHeight)
284                {
285                        return new Point2dImpl(pt);
286                }
287
288                // otherwise return the updated point
289                return current;
290        }
291}