001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.feature.local.interest; 031 032import java.util.ArrayList; 033import java.util.List; 034 035import odk.lang.FastMath; 036 037import org.openimaj.algorithm.iterative.IterationState; 038import org.openimaj.image.FImage; 039import org.openimaj.image.processing.convolution.FImageConvolveSeparable; 040import org.openimaj.math.geometry.point.Point2d; 041import org.openimaj.math.geometry.point.Point2dImpl; 042import org.openimaj.math.matrix.PseudoInverse; 043import org.openimaj.util.function.Predicate; 044 045import Jama.Matrix; 046 047/** 048 * Refines detected corners (i.e. from {@link HarrisIPD} or other methods) to an 049 * optimised sub-pixel estimate. The method works by iteratively updating the 050 * sub-pixel position of a point based on the inverse of the local gradient 051 * auto-correlation matrix. 052 * 053 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 054 */ 055public class SubPixelCorners { 056 private static final float GRAD_X_KERNEL[] = { -1.f, 0.f, 1.f }; 057 private static final float GRAD_Y_KERNEL[] = { 0.f, 0.5f, 0.f }; 058 059 private Predicate<IterationState> iter; 060 private int halfWidth; 061 private int halfHeight; 062 private int zeroZoneHalfWidth = -1; 063 private int zeroZoneHalfHeight = -1; 064 065 /** 066 * Construct with the given search window size and predicate to <b>stop</b> 067 * the iteration. No zeroing of weights is performed 068 * 069 * @param halfWidth 070 * half the search window width (actual size is 2*halfWidth + 1, 071 * centred on point) 072 * @param halfHeight 073 * half the search window height (actual size is 2*halfHeight + 074 * 1, centred on point) 075 * @param iter 076 * the predicate to stop iteration 077 */ 078 public SubPixelCorners(int halfWidth, int halfHeight, Predicate<IterationState> iter) { 079 this.halfWidth = halfWidth; 080 this.halfHeight = halfHeight; 081 this.iter = iter; 082 } 083 084 /** 085 * Construct with the given search window size, zeroed window and predicate 086 * to <b>stop</b> the iteration. 087 * <p> 088 * The zeroed window is a dead region in the middle of the search zone over 089 * which the summation over gradients is not done. It is used sometimes to 090 * avoid possible singularities of the autocorrelation matrix. 091 * 092 * @param halfWidth 093 * half the search window width (actual size is 2*halfWidth + 1, 094 * centred on point) 095 * @param halfHeight 096 * half the search window height (actual size is 2*halfHeight + 097 * 1, centred on point) 098 * @param zeroZoneHalfWidth 099 * the half-width of the zeroed region in the weighting array 100 * @param zeroZoneHalfHeight 101 * the half-height of the zeroed region in the weighting array 102 * @param iter 103 * the predicate to stop iteration 104 */ 105 public SubPixelCorners(int halfWidth, int halfHeight, int zeroZoneHalfWidth, int zeroZoneHalfHeight, 106 Predicate<IterationState> iter) 107 { 108 this.halfWidth = halfWidth; 109 this.halfHeight = halfHeight; 110 this.zeroZoneHalfWidth = zeroZoneHalfWidth; 111 this.zeroZoneHalfHeight = zeroZoneHalfHeight; 112 this.iter = iter; 113 } 114 115 /** 116 * Find the sub-pixel estimated position of each corner 117 * 118 * @param src 119 * the image 120 * @param corners 121 * the initial corner positions 122 * @return the updated corners 123 */ 124 public List<Point2dImpl> findSubPixCorners(FImage src, List<? extends Point2d> corners) 125 { 126 final List<Point2dImpl> outCorners = new ArrayList<Point2dImpl>(corners.size()); 127 final int windowWidth = halfWidth * 2 + 1; 128 final int windowHeight = halfHeight * 2 + 1; 129 130 if (corners.size() == 0) 131 return outCorners; 132 133 final FImage weights = this.buildGaussianWeights(windowWidth, windowHeight); 134 final FImage gx = new FImage(windowWidth, windowHeight); 135 final FImage gy = new FImage(windowWidth, windowHeight); 136 final float[] buffer = new float[windowWidth + 2]; 137 138 // note 2px padding as conv reduces size: 139 final FImage roi = new FImage(windowWidth + 2, windowHeight + 2); 140 141 // loop for all the points 142 for (int i = 0; i < corners.size(); i++) 143 { 144 final Point2d pt = corners.get(i); 145 outCorners.add(this.findCornerSubPix(src, pt, roi, gx, gy, weights, buffer)); 146 } 147 148 return outCorners; 149 } 150 151 /** 152 * Find the sub-pixel estimated position of a corner 153 * 154 * @param src 155 * the image 156 * @param corner 157 * the initial corner position 158 * @return the updated corner position 159 */ 160 public Point2dImpl findSubPixCorner(FImage src, Point2d corner) 161 { 162 final int windowWidth = halfWidth * 2 + 1; 163 final int windowHeight = halfHeight * 2 + 1; 164 165 final FImage weights = this.buildGaussianWeights(windowWidth, windowHeight); 166 final FImage gx = new FImage(windowWidth, windowHeight); 167 final FImage gy = new FImage(windowWidth, windowHeight); 168 final float[] buffer = new float[windowWidth + 2]; 169 170 // note 2px padding as conv reduces size: 171 final FImage roi = new FImage(windowWidth + 2, windowHeight + 2); 172 173 return this.findCornerSubPix(src, corner, roi, gx, gy, weights, buffer); 174 } 175 176 /** 177 * Build the Gaussian weighting image 178 * 179 * @param width 180 * the width 181 * @param height 182 * the height 183 * @return 184 */ 185 private FImage buildGaussianWeights(int width, int height) { 186 final FImage weights = new FImage(width, height); 187 final float[] weightsX = new float[width]; 188 189 double coeff = 1. / (halfWidth * halfWidth); 190 for (int i = -halfWidth, k = 0; i <= halfWidth; i++, k++) 191 { 192 weightsX[k] = (float) Math.exp(-i * i * coeff); 193 } 194 195 float[] weightsY; 196 if (halfWidth == halfHeight) { 197 weightsY = weightsX; 198 } else { 199 weightsY = new float[height]; 200 coeff = 1. / (halfHeight * halfHeight); 201 for (int i = -halfHeight, k = 0; i <= halfHeight; i++, k++) 202 { 203 weightsY[k] = (float) Math.exp(-i * i * coeff); 204 } 205 } 206 207 for (int i = 0; i < height; i++) { 208 for (int j = 0; j < width; j++) { 209 weights.pixels[i][j] = weightsX[j] * weightsY[i]; 210 } 211 } 212 213 // if necessary, mask off the centre portion 214 if (zeroZoneHalfWidth >= 0 && zeroZoneHalfHeight >= 0 && 215 zeroZoneHalfWidth * 2 + 1 < width && zeroZoneHalfHeight * 2 + 1 < height) 216 { 217 for (int i = halfHeight - zeroZoneHalfHeight; i <= halfHeight + zeroZoneHalfHeight; i++) { 218 for (int j = halfWidth - zeroZoneHalfWidth; j <= halfWidth + zeroZoneHalfWidth; j++) { 219 weights.pixels[i][j] = 0; 220 } 221 } 222 } 223 224 return weights; 225 } 226 227 private Point2dImpl findCornerSubPix(FImage src, Point2d pt, FImage roi, FImage gx, FImage gy, FImage weights, 228 final float[] buffer) 229 { 230 final IterationState state = new IterationState(); 231 Point2dImpl current = new Point2dImpl(pt); 232 233 while (!iter.test(state)) { 234 // get window around pt 235 src.extractCentreSubPix(current, roi); 236 237 // calc derivatives 238 FImageConvolveSeparable.fastConvolve3(roi, gx, GRAD_X_KERNEL, GRAD_Y_KERNEL, buffer); 239 FImageConvolveSeparable.fastConvolve3(roi, gy, GRAD_Y_KERNEL, GRAD_X_KERNEL, buffer); 240 241 double a = 0, b = 0, c = 0, bb1 = 0, bb2 = 0; 242 final int win_w = weights.width; 243 final int win_h = weights.height; 244 245 // process gradient 246 for (int i = 0; i < win_h; i++) 247 { 248 final double py = i - halfHeight; 249 250 for (int j = 0; j < win_w; j++) 251 { 252 final double m = weights.pixels[i][j]; 253 final double tgx = gx.pixels[i][j]; 254 final double tgy = gy.pixels[i][j]; 255 final double gxx = tgx * tgx * m; 256 final double gxy = tgx * tgy * m; 257 final double gyy = tgy * tgy * m; 258 final double px = j - halfWidth; 259 260 a += gxx; 261 b += gxy; 262 c += gyy; 263 264 bb1 += gxx * px + gxy * py; 265 bb2 += gxy * px + gyy * py; 266 } 267 } 268 269 final Matrix m = new Matrix(new double[][] { { a, b }, { b, c } }); 270 final Matrix mInv = PseudoInverse.pseudoInverse(m); 271 final Point2dImpl cI2 = new Point2dImpl(); 272 cI2.x = (float) (current.x + mInv.get(0, 0) * bb1 + mInv.get(0, 1) * bb2); 273 cI2.y = (float) (current.y + mInv.get(1, 0) * bb1 + mInv.get(1, 1) * bb2); 274 275 state.epsilon = FastMath.sqrt((cI2.x - current.x) * (cI2.x - current.x) + (cI2.y - current.y) 276 * (cI2.y - current.y)); 277 state.iteration++; 278 current = cI2; 279 } 280 281 // if new point is too far from initial, it means poor convergence. 282 // return initial point as the result 283 if (Math.abs(current.x - pt.getX()) > halfWidth || Math.abs(current.y - pt.getY()) > halfHeight) 284 { 285 return new Point2dImpl(pt); 286 } 287 288 // otherwise return the updated point 289 return current; 290 } 291}