001/** 002 * Copyright (c) 2011, The University of Southampton and the individual contributors. 003 * All rights reserved. 004 * 005 * Redistribution and use in source and binary forms, with or without modification, 006 * are permitted provided that the following conditions are met: 007 * 008 * * Redistributions of source code must retain the above copyright notice, 009 * this list of conditions and the following disclaimer. 010 * 011 * * Redistributions in binary form must reproduce the above copyright notice, 012 * this list of conditions and the following disclaimer in the documentation 013 * and/or other materials provided with the distribution. 014 * 015 * * Neither the name of the University of Southampton nor the names of its 016 * contributors may be used to endorse or promote products derived from this 017 * software without specific prior written permission. 018 * 019 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 022 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 023 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 024 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 025 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 026 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 027 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 028 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 029 */ 030package org.openimaj.image.processing.face.detection; 031 032import java.io.ByteArrayInputStream; 033import java.io.ByteArrayOutputStream; 034import java.io.DataInput; 035import java.io.DataOutput; 036import java.io.File; 037import java.io.IOException; 038import java.io.ObjectInputStream; 039import java.io.ObjectOutputStream; 040import java.util.ArrayList; 041import java.util.List; 042 043import org.openimaj.citation.annotation.Reference; 044import org.openimaj.citation.annotation.ReferenceType; 045import org.openimaj.image.FImage; 046import org.openimaj.image.ImageUtilities; 047import org.openimaj.image.MBFImage; 048import org.openimaj.image.colour.Transforms; 049import org.openimaj.image.connectedcomponent.ConnectedComponentLabeler; 050import org.openimaj.image.model.pixel.HistogramPixelModel; 051import org.openimaj.image.model.pixel.MBFPixelClassificationModel; 052import org.openimaj.image.pixel.ConnectedComponent; 053import org.openimaj.image.pixel.ConnectedComponent.ConnectMode; 054import org.openimaj.image.processing.convolution.FSobelMagnitude; 055import org.openimaj.image.processor.connectedcomponent.render.OrientatedBoundingBoxRenderer; 056import org.openimaj.math.geometry.shape.Rectangle; 057 058/** 059 * Implementation of a face detector along the lines of 060 * "Human Face Detection in Cluttered Color Images Using Skin Color and Edge Information" 061 * K. Sandeep and A. N. Rajagopalan (IIT/Madras) 062 * 063 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk) 064 * 065 */ 066@Reference( 067 type = ReferenceType.Article, 068 author = { "Sandeep, K", "Rajagopalan, A N" }, 069 title = "Human Face Detection in Cluttered Color Images Using Skin Color and Edge Information", 070 year = "2002", 071 journal = "Electrical Engineering", 072 url = "http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.12.730&rep=rep1&type=pdf", 073 publisher = "Citeseer") 074public class SandeepFaceDetector implements FaceDetector<CCDetectedFace, MBFImage> { 075 /** 076 * The golden ratio (for comparing facial height/width) 077 */ 078 public final static double GOLDEN_RATIO = 1.618033989; // ((1 + sqrt(5) / 2) 079 080 private final String DEFAULT_MODEL = "/org/openimaj/image/processing/face/detection/skin-histogram-16-6.bin"; 081 082 private ConnectedComponentLabeler ccl; 083 084 MBFPixelClassificationModel skinModel; 085 float skinThreshold = 0.1F; 086 float edgeThreshold = 125F / 255F; 087 float goldenRatioThreshold = 0.65F; 088 float percentageThreshold = 0.55F; 089 090 /** 091 * Construct a new {@link SandeepFaceDetector} with the default skin-tone 092 * model. 093 */ 094 public SandeepFaceDetector() { 095 ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_8); 096 097 try { 098 if (this.getClass().getResource(DEFAULT_MODEL) == null) { 099 // This is to create the skin model 100 skinModel = new HistogramPixelModel(16, 6); 101 final MBFImage rgb = ImageUtilities.readMBF(this.getClass().getResourceAsStream("skin.png")); 102 skinModel.learnModel(Transforms.RGB_TO_HS(rgb)); 103 // ObjectOutputStream oos = new ObjectOutputStream(new 104 // FileOutputStream(new 105 // File("D:\\Programming\\skin-histogram-16-6.bin"))); 106 // oos.writeObject(skinModel); 107 } else { 108 // Load in the skin model 109 final ObjectInputStream ois = new ObjectInputStream(this.getClass().getResourceAsStream(DEFAULT_MODEL)); 110 skinModel = (MBFPixelClassificationModel) ois.readObject(); 111 } 112 } catch (final Exception e) { 113 e.printStackTrace(); 114 throw new RuntimeException(e); 115 } 116 } 117 118 /** 119 * Construct the detector with the given pixel classification model. 120 * 121 * @param skinModel 122 * the underlying classification model. 123 */ 124 public SandeepFaceDetector(MBFPixelClassificationModel skinModel) { 125 ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_8); 126 this.skinModel = skinModel; 127 } 128 129 protected FImage generateSkinColorMap(MBFImage inputHS) { 130 final FImage map = skinModel.predict(inputHS); 131 132 map.clipMin(skinThreshold); 133 return map; 134 } 135 136 protected FImage generateSobelMagnitudes(MBFImage inputRGB) { 137 final MBFImage mag = inputRGB.process(new FSobelMagnitude()); 138 final FImage ret = mag.flattenMax().clipMax(edgeThreshold); 139 return ret; 140 } 141 142 protected FImage generateFaceMap(FImage skin, FImage edge) { 143 for (int y = 0; y < skin.height; y++) { 144 for (int x = 0; x < skin.height; x++) { 145 146 if (edge.pixels[y][x] != 0 && skin.pixels[y][x] != 0) 147 skin.pixels[y][x] = 1f; 148 else 149 skin.pixels[y][x] = 0f; 150 } 151 } 152 153 return skin; 154 } 155 156 protected List<CCDetectedFace> extractFaces(FImage faceMap, FImage skinMap, FImage image) { 157 final List<ConnectedComponent> blobs = ccl.findComponents(faceMap); 158 final List<CCDetectedFace> faces = new ArrayList<CCDetectedFace>(); 159 160 for (final ConnectedComponent blob : blobs) { 161 if (blob.calculateArea() > 1000) { 162 final double[] centroid = blob.calculateCentroid(); 163 final double[] hw = blob.calculateAverageHeightWidth(centroid); 164 165 final double percentageSkin = calculatePercentageSkin(skinMap, 166 (int) Math.round(centroid[0] - (hw[0] / 2)), 167 (int) Math.round(centroid[1] - (hw[1] / 2)), 168 (int) Math.round(centroid[0] + (hw[0] / 2)), 169 (int) Math.round(centroid[1] + (hw[1] / 2))); 170 171 final double ratio = hw[0] / hw[1]; 172 173 if (Math.abs(ratio - GOLDEN_RATIO) < goldenRatioThreshold && percentageSkin > percentageThreshold) { 174 final Rectangle r = blob.calculateRegularBoundingBox(); 175 faces.add(new CCDetectedFace( 176 r, 177 image.extractROI(r), 178 blob, 179 (float) ((percentageSkin / percentageThreshold) * (Math.abs(ratio - GOLDEN_RATIO) / goldenRatioThreshold)))); 180 } 181 } 182 } 183 184 return faces; 185 } 186 187 private double calculatePercentageSkin(FImage skinMap, int l, int t, int r, int b) { 188 int npix = 0; 189 int nskin = 0; 190 191 l = Math.max(l, 0); 192 t = Math.max(t, 0); 193 r = Math.min(r, skinMap.getWidth()); 194 b = Math.min(b, skinMap.getHeight()); 195 196 for (int y = t; y < b; y++) { 197 for (int x = l; x < r; x++) { 198 npix++; 199 if (skinMap.pixels[y][x] != 0) 200 nskin++; 201 } 202 } 203 204 return (double) nskin / (double) npix; 205 } 206 207 @Override 208 public List<CCDetectedFace> detectFaces(MBFImage inputRGB) { 209 final FImage skin = generateSkinColorMap(Transforms.RGB_TO_HS(inputRGB)); 210 final FImage edge = generateSobelMagnitudes(inputRGB); 211 212 final FImage map = generateFaceMap(skin, edge); 213 214 return extractFaces(map, skin, Transforms.calculateIntensityNTSC(inputRGB)); 215 } 216 217 /** 218 * @return The underlying skin-tone classifier 219 */ 220 public MBFPixelClassificationModel getSkinModel() { 221 return skinModel; 222 } 223 224 /** 225 * Set the underlying skin-tone classifier 226 * 227 * @param skinModel 228 */ 229 public void setSkinModel(MBFPixelClassificationModel skinModel) { 230 this.skinModel = skinModel; 231 } 232 233 /** 234 * @return the detection threshold. 235 */ 236 public float getSkinThreshold() { 237 return skinThreshold; 238 } 239 240 /** 241 * Set the detection threshold. 242 * 243 * @param skinThreshold 244 */ 245 public void setSkinThreshold(float skinThreshold) { 246 this.skinThreshold = skinThreshold; 247 } 248 249 /** 250 * @return The edge threshold. 251 */ 252 public float getEdgeThreshold() { 253 return edgeThreshold; 254 } 255 256 /** 257 * Set the edge threshold. 258 * 259 * @param edgeThreshold 260 */ 261 public void setEdgeThreshold(float edgeThreshold) { 262 this.edgeThreshold = edgeThreshold; 263 } 264 265 /** 266 * @return The percentage threshold 267 */ 268 public float getPercentageThreshold() { 269 return percentageThreshold; 270 } 271 272 /** 273 * Set the percentage threshold 274 * 275 * @param percentageThreshold 276 */ 277 public void setPercentageThreshold(float percentageThreshold) { 278 this.percentageThreshold = percentageThreshold; 279 } 280 281 /** 282 * Run the face detector following the conventions of the ocv detector 283 * 284 * @param args 285 * @throws IOException 286 */ 287 public static void main(String[] args) throws IOException { 288 if (args.length < 1 || args.length > 2) { 289 System.err.println("Usage: SandeepFaceDetector filename [filename_out]"); 290 return; 291 } 292 293 final String inputImage = args[0]; 294 String outputImage = null; 295 if (args.length == 2) 296 outputImage = args[1]; 297 298 final SandeepFaceDetector sfd = new SandeepFaceDetector(); 299 300 // tweek the default settings 301 sfd.edgeThreshold = 0.39F; 302 sfd.ccl = new ConnectedComponentLabeler(ConnectMode.CONNECT_4); 303 304 final MBFImage image = ImageUtilities.readMBF(new File(inputImage)); 305 final List<CCDetectedFace> faces = sfd.detectFaces(image); 306 307 if (outputImage != null) { 308 final OrientatedBoundingBoxRenderer<Float> render = new OrientatedBoundingBoxRenderer<Float>( 309 image.getWidth(), image.getHeight(), 1.0F); 310 for (final CCDetectedFace f : faces) 311 f.connectedComponent.process(render); 312 image.multiplyInplace(render.getImage().inverse()); 313 314 ImageUtilities.write(image, outputImage.substring(outputImage.lastIndexOf('.') + 1), new File(outputImage)); 315 } 316 317 for (final CCDetectedFace f : faces) { 318 System.out.format("%s, %d, %d, %d, %d\n", 319 "uk.ac.soton.ecs.jsh2.image.proc.tools.face.detection.skin-histogram-16-6.bin", 320 f.bounds.x, 321 f.bounds.y, 322 f.bounds.width, 323 f.bounds.height 324 ); 325 } 326 } 327 328 @Override 329 public void readBinary(DataInput in) throws IOException { 330 // ccl; 331 332 try { 333 final byte[] bytes = new byte[in.readInt()]; 334 in.readFully(bytes); 335 skinModel = (MBFPixelClassificationModel) new ObjectInputStream(new ByteArrayInputStream(bytes)).readObject(); 336 } catch (final ClassNotFoundException e) { 337 throw new IOException(e); 338 } 339 340 skinThreshold = in.readFloat(); 341 edgeThreshold = in.readFloat(); 342 goldenRatioThreshold = in.readFloat(); 343 percentageThreshold = in.readFloat(); 344 } 345 346 @Override 347 public byte[] binaryHeader() { 348 return "SdFD".getBytes(); 349 } 350 351 @Override 352 public void writeBinary(DataOutput out) throws IOException { 353 // ccl; 354 355 final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 356 final ObjectOutputStream oos = new ObjectOutputStream(baos); 357 oos.writeObject(skinModel); 358 oos.close(); 359 360 out.writeInt(baos.size()); 361 out.write(baos.toByteArray()); 362 363 out.writeFloat(skinThreshold); 364 out.writeFloat(edgeThreshold); 365 out.writeFloat(goldenRatioThreshold); 366 out.writeFloat(percentageThreshold); 367 } 368}